From ceda85f60112b418df029060c22e0925d2a45964 Mon Sep 17 00:00:00 2001 From: Alan Davis Date: Tue, 10 Sep 2019 10:34:24 +0100 Subject: [PATCH] Revert "The Tika T-Engine was not being called by the Transform Service or Local transforms, because the sub-transform name was required." This reverts commit 7f1a151ca4722af234ea9945d19cb6c2bbc34735. --- pom.xml | 10 +- .../transform/ArchiveContentTransformer.java | 9 +- .../content/transform/LocalTransformImpl.java | 2 +- .../LocalTransformServiceRegistry.java | 8 +- .../transform/MailContentTransformer.java | 6 + .../transform/PdfBoxContentTransformer.java | 10 + .../transform/PoiContentTransformer.java | 6 + .../transform/PoiHssfContentTransformer.java | 6 + .../transform/PoiOOXMLContentTransformer.java | 6 + .../TextMiningContentTransformer.java | 6 + .../transform/TikaAutoContentTransformer.java | 6 + .../TikaPoweredContentTransformer.java | 6 +- ...ikaSpringConfiguredContentTransformer.java | 6 + .../AbstractTransformServiceRegistry.java | 6 - .../client/model/config/ChildTransformer.java | 66 ++ .../client/model/config/CombinedConfig.java | 264 ++++++- .../config/TransformServiceRegistry.java | 67 ++ .../config/TransformServiceRegistryImpl.java | 303 +++++++- ...calTransformServiceRegistryConfigTest.java | 120 ++-- .../client/model/config/TransformBuilder.java | 54 ++ .../TransformServiceRegistryConfigTest.java | 661 +++++++++++++++++- 21 files changed, 1478 insertions(+), 150 deletions(-) create mode 100644 src/main/java/org/alfresco/transform/client/model/config/ChildTransformer.java create mode 100644 src/main/java/org/alfresco/transform/client/model/config/TransformServiceRegistry.java create mode 100644 src/test/java/org/alfresco/transform/client/model/config/TransformBuilder.java diff --git a/pom.xml b/pom.xml index 16568b6035..6a6d06c573 100644 --- a/pom.xml +++ b/pom.xml @@ -44,7 +44,7 @@ 7.1 1.1 1.0.11 - repo4639.1 + 5.1.8.RELEASE 4.5.9 @@ -66,6 +66,7 @@ 3.3.2 2.9.9 2.9.9.3 + 1.0.2.5 @@ -1051,13 +1052,6 @@ 4.12 test - - org.alfresco - alfresco-transform-model - ${dependency.transform.model.version} - tests - test - org.alfresco alfresco-core diff --git a/src/main/java/org/alfresco/repo/content/transform/ArchiveContentTransformer.java b/src/main/java/org/alfresco/repo/content/transform/ArchiveContentTransformer.java index fba7289eee..cfa5c27fc3 100644 --- a/src/main/java/org/alfresco/repo/content/transform/ArchiveContentTransformer.java +++ b/src/main/java/org/alfresco/repo/content/transform/ArchiveContentTransformer.java @@ -141,6 +141,12 @@ public class ArchiveContentTransformer extends TikaPoweredContentTransformer return context; } + @Override + protected String getTransform() + { + return "Archive"; + } + @Override protected void transformRemote(RemoteTransformerClient remoteTransformerClient, ContentReader reader, ContentWriter writer, TransformationOptions options, @@ -148,6 +154,7 @@ public class ArchiveContentTransformer extends TikaPoweredContentTransformer String sourceExtension, String targetExtension, String targetEncoding) throws Exception { + String transform = getTransform(); long timeoutMs = options.getTimeoutMs(); boolean recurse = includeContents; if(options.getIncludeEmbedded() != null) @@ -155,7 +162,7 @@ public class ArchiveContentTransformer extends TikaPoweredContentTransformer recurse = options.getIncludeEmbedded(); } remoteTransformerClient.request(reader, writer, sourceMimetype, sourceExtension, targetExtension, - timeoutMs, logger, "includeContents", Boolean.toString(recurse), + timeoutMs, logger, "transform", transform, "includeContents", Boolean.toString(recurse), "targetMimetype", targetMimetype, "targetEncoding", targetEncoding); } } diff --git a/src/main/java/org/alfresco/repo/content/transform/LocalTransformImpl.java b/src/main/java/org/alfresco/repo/content/transform/LocalTransformImpl.java index 08f78e443b..3611fe860b 100644 --- a/src/main/java/org/alfresco/repo/content/transform/LocalTransformImpl.java +++ b/src/main/java/org/alfresco/repo/content/transform/LocalTransformImpl.java @@ -145,7 +145,7 @@ public class LocalTransformImpl extends AbstractLocalTransform { // At some point in the future, we may decide to only pass the sourceEncoding and other dynamic values like // it if they were supplied in the rendition definition without a value. The sourceEncoding value is also - // supplied in the RenditionEventProducer in the message to the T-Router. + // supplied in the TransformRequest (message to the T-Router). transformOptions = new HashMap<>(transformOptions); if (transformOptions.get(SOURCE_ENCODING) == null) { diff --git a/src/main/java/org/alfresco/repo/content/transform/LocalTransformServiceRegistry.java b/src/main/java/org/alfresco/repo/content/transform/LocalTransformServiceRegistry.java index 05e48cf0f4..e0fd49a52e 100644 --- a/src/main/java/org/alfresco/repo/content/transform/LocalTransformServiceRegistry.java +++ b/src/main/java/org/alfresco/repo/content/transform/LocalTransformServiceRegistry.java @@ -30,11 +30,10 @@ import org.alfresco.service.cmr.repository.ContentWriter; import org.alfresco.service.cmr.repository.MimetypeService; import org.alfresco.service.cmr.repository.NodeRef; import org.alfresco.transform.client.model.config.CombinedConfig; -import org.alfresco.transform.client.model.config.TransformOption; +import org.alfresco.transform.client.model.config.InlineTransformer; import org.alfresco.transform.client.model.config.TransformServiceRegistry; import org.alfresco.transform.client.model.config.TransformServiceRegistryImpl; import org.alfresco.transform.client.model.config.TransformStep; -import org.alfresco.transform.client.model.config.Transformer; import org.alfresco.util.PropertyCheck; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -157,8 +156,7 @@ public class LocalTransformServiceRegistry extends TransformServiceRegistryImpl } @Override - protected void register(Transformer transformer, Map> transformOptions, - String baseUrl, String readFrom) + protected void register(InlineTransformer transformer, String baseUrl, String readFrom) { try { @@ -244,7 +242,7 @@ public class LocalTransformServiceRegistry extends TransformServiceRegistryImpl } } localTransforms.put(name, localTransform); - super.register(transformer, transformOptions, baseUrl, readFrom); + super.register(transformer, baseUrl, readFrom); } catch (IllegalArgumentException e) { diff --git a/src/main/java/org/alfresco/repo/content/transform/MailContentTransformer.java b/src/main/java/org/alfresco/repo/content/transform/MailContentTransformer.java index 2e28422367..9caffe2a5c 100644 --- a/src/main/java/org/alfresco/repo/content/transform/MailContentTransformer.java +++ b/src/main/java/org/alfresco/repo/content/transform/MailContentTransformer.java @@ -51,4 +51,10 @@ public class MailContentTransformer extends TikaPoweredContentTransformer protected Parser getParser() { return new OfficeParser(); } + + @Override + protected String getTransform() + { + return "OutlookMsg"; + } } diff --git a/src/main/java/org/alfresco/repo/content/transform/PdfBoxContentTransformer.java b/src/main/java/org/alfresco/repo/content/transform/PdfBoxContentTransformer.java index 7534cd8fe9..25158a36d7 100644 --- a/src/main/java/org/alfresco/repo/content/transform/PdfBoxContentTransformer.java +++ b/src/main/java/org/alfresco/repo/content/transform/PdfBoxContentTransformer.java @@ -98,6 +98,13 @@ public class PdfBoxContentTransformer extends TikaPoweredContentTransformer return context; } + @Override + protected String getTransform() + { + return "PdfBox"; + } + + @Override protected void transformRemote(RemoteTransformerClient remoteTransformerClient, ContentReader reader, ContentWriter writer, TransformationOptions options, @@ -105,6 +112,8 @@ public class PdfBoxContentTransformer extends TikaPoweredContentTransformer String sourceExtension, String targetExtension, String targetEncoding) throws Exception { + + String transform = getTransform(); long timeoutMs = options.getTimeoutMs(); String notExtractBookmarksText = null; @@ -115,6 +124,7 @@ public class PdfBoxContentTransformer extends TikaPoweredContentTransformer remoteTransformerClient.request(reader, writer, sourceMimetype, sourceExtension, targetExtension, timeoutMs, logger, + "transform", transform, "notExtractBookmarksText", notExtractBookmarksText, "targetMimetype", targetMimetype, "targetEncoding", targetEncoding); diff --git a/src/main/java/org/alfresco/repo/content/transform/PoiContentTransformer.java b/src/main/java/org/alfresco/repo/content/transform/PoiContentTransformer.java index 62e8d70e30..9b4cc4897a 100644 --- a/src/main/java/org/alfresco/repo/content/transform/PoiContentTransformer.java +++ b/src/main/java/org/alfresco/repo/content/transform/PoiContentTransformer.java @@ -76,4 +76,10 @@ public class PoiContentTransformer extends TikaPoweredContentTransformer protected Parser getParser() { return new OfficeParser(); } + + @Override + protected String getTransform() + { + return "Office"; + } } diff --git a/src/main/java/org/alfresco/repo/content/transform/PoiHssfContentTransformer.java b/src/main/java/org/alfresco/repo/content/transform/PoiHssfContentTransformer.java index f8ca2d0b88..aefceef32a 100644 --- a/src/main/java/org/alfresco/repo/content/transform/PoiHssfContentTransformer.java +++ b/src/main/java/org/alfresco/repo/content/transform/PoiHssfContentTransformer.java @@ -211,4 +211,10 @@ public class PoiHssfContentTransformer extends TikaPoweredContentTransformer } } } + + @Override + protected String getTransform() + { + return "Poi"; + } } diff --git a/src/main/java/org/alfresco/repo/content/transform/PoiOOXMLContentTransformer.java b/src/main/java/org/alfresco/repo/content/transform/PoiOOXMLContentTransformer.java index 38f3a3e4c3..fb1e076a9c 100644 --- a/src/main/java/org/alfresco/repo/content/transform/PoiOOXMLContentTransformer.java +++ b/src/main/java/org/alfresco/repo/content/transform/PoiOOXMLContentTransformer.java @@ -65,4 +65,10 @@ public class PoiOOXMLContentTransformer extends TikaPoweredContentTransformer protected Parser getParser() { return new OOXMLParser(); } + + @Override + protected String getTransform() + { + return "OOXML"; + } } diff --git a/src/main/java/org/alfresco/repo/content/transform/TextMiningContentTransformer.java b/src/main/java/org/alfresco/repo/content/transform/TextMiningContentTransformer.java index d2505fff14..0b40d1ec80 100644 --- a/src/main/java/org/alfresco/repo/content/transform/TextMiningContentTransformer.java +++ b/src/main/java/org/alfresco/repo/content/transform/TextMiningContentTransformer.java @@ -51,4 +51,10 @@ public class TextMiningContentTransformer extends TikaPoweredContentTransformer protected Parser getParser() { return new OfficeParser(); } + + @Override + protected String getTransform() + { + return "TextMining"; + } } diff --git a/src/main/java/org/alfresco/repo/content/transform/TikaAutoContentTransformer.java b/src/main/java/org/alfresco/repo/content/transform/TikaAutoContentTransformer.java index 3b52f097bb..39370059b3 100644 --- a/src/main/java/org/alfresco/repo/content/transform/TikaAutoContentTransformer.java +++ b/src/main/java/org/alfresco/repo/content/transform/TikaAutoContentTransformer.java @@ -131,4 +131,10 @@ public class TikaAutoContentTransformer extends TikaPoweredContentTransformer { return parser; } + + @Override + protected String getTransform() + { + return "TikaAuto"; + } } diff --git a/src/main/java/org/alfresco/repo/content/transform/TikaPoweredContentTransformer.java b/src/main/java/org/alfresco/repo/content/transform/TikaPoweredContentTransformer.java index 4ee1df393b..20f80ad286 100644 --- a/src/main/java/org/alfresco/repo/content/transform/TikaPoweredContentTransformer.java +++ b/src/main/java/org/alfresco/repo/content/transform/TikaPoweredContentTransformer.java @@ -292,14 +292,18 @@ public abstract class TikaPoweredContentTransformer extends AbstractRemoteConten String sourceExtension, String targetExtension, String targetEncoding) throws Exception { + String transform = getTransform(); long timeoutMs = options.getTimeoutMs(); remoteTransformerClient.request(reader, writer, sourceMimetype, sourceExtension, targetExtension, timeoutMs, logger, - "targetMimetype", targetMimetype, + "transform", transform, + "targetMimetype", targetMimetype, "targetEncoding", targetEncoding); } + protected abstract String getTransform(); + private String calculateMemoryAndTimeUsage(ContentReader reader, long startTime) { long endTime = System.currentTimeMillis(); diff --git a/src/main/java/org/alfresco/repo/content/transform/TikaSpringConfiguredContentTransformer.java b/src/main/java/org/alfresco/repo/content/transform/TikaSpringConfiguredContentTransformer.java index 63f9bbee1c..5aded4859f 100644 --- a/src/main/java/org/alfresco/repo/content/transform/TikaSpringConfiguredContentTransformer.java +++ b/src/main/java/org/alfresco/repo/content/transform/TikaSpringConfiguredContentTransformer.java @@ -107,4 +107,10 @@ public class TikaSpringConfiguredContentTransformer extends TikaPoweredContentTr throw new AlfrescoRuntimeException("Unable to create specified Parser", e); } } + + @Override + protected String getTransform() + { + return "tikaspring"; // This transformer is no longer created by Spring, so is not supported by the Tika transformer image + } } diff --git a/src/main/java/org/alfresco/repo/rendition2/AbstractTransformServiceRegistry.java b/src/main/java/org/alfresco/repo/rendition2/AbstractTransformServiceRegistry.java index efc57d1582..4a7ae5a5f4 100644 --- a/src/main/java/org/alfresco/repo/rendition2/AbstractTransformServiceRegistry.java +++ b/src/main/java/org/alfresco/repo/rendition2/AbstractTransformServiceRegistry.java @@ -42,10 +42,4 @@ public abstract class AbstractTransformServiceRegistry implements TransformServi long maxSize = getMaxSize(sourceMimetype, targetMimetype, options, renditionName); return maxSize != 0 && (maxSize == -1L || maxSize >= size); } - - @Override - public String getTransformerName(String sourceMimetype, long sourceSizeInBytes, String targetMimetype, Map actualOptions, String renditionName) - { - throw new UnsupportedOperationException("AbstractTransformServiceRegistry.getTransformerName(...) is not supported. Only supported in "); - } } diff --git a/src/main/java/org/alfresco/transform/client/model/config/ChildTransformer.java b/src/main/java/org/alfresco/transform/client/model/config/ChildTransformer.java new file mode 100644 index 0000000000..382f7b3ddd --- /dev/null +++ b/src/main/java/org/alfresco/transform/client/model/config/ChildTransformer.java @@ -0,0 +1,66 @@ +/* + * #%L + * Alfresco Repository + * %% + * Copyright (C) 2005 - 2018 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.client.model.config; + +/** + * Represents a single transformer in a pipeline of multiple transformers. A transformer's options may be optional or + * required in the containing transformer. Historically in ACS only options for the final transformer were provided. + */ +public class ChildTransformer +{ + private boolean required; + private InlineTransformer transformer; + + public ChildTransformer() + { + } + + public ChildTransformer(boolean required, InlineTransformer transformer) + { + this.required = required; + this.transformer = transformer; + } + + public boolean isRequired() + { + return required; + } + + public void setRequired(boolean required) + { + this.required = required; + } + + public InlineTransformer getTransformer() + { + return transformer; + } + + public void setTransformer(InlineTransformer transformer) + { + this.transformer = transformer; + } +} diff --git a/src/main/java/org/alfresco/transform/client/model/config/CombinedConfig.java b/src/main/java/org/alfresco/transform/client/model/config/CombinedConfig.java index bb02acea8f..dbeb3f5b79 100644 --- a/src/main/java/org/alfresco/transform/client/model/config/CombinedConfig.java +++ b/src/main/java/org/alfresco/transform/client/model/config/CombinedConfig.java @@ -27,6 +27,8 @@ package org.alfresco.transform.client.model.config; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.alfresco.error.AlfrescoRuntimeException; import org.alfresco.util.ConfigFileFinder; import org.apache.commons.logging.Log; @@ -42,45 +44,65 @@ import java.io.IOException; import java.io.StringReader; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; /** - * This class reads multiple T-Engine config and local files and registers them all with a registry as if they were all - * in one file. Transform options are shared between all sources.

+ * This class recreates the json format used in ACS 6.1 where we just had an array of transformers and each + * transformer has a list of node options. The idea of this code is that it replaces the references with the + * actual node options that have been separated out into their own section.

* - * The caller should make calls to {@link #addRemoteConfig(List, String)} and {@link #addLocalConfig(String)} followed - * by a call to {@link #register(TransformServiceRegistryImpl)}. - * - * @author adavis + * The T-Router and T-Engines return the format with the node option separated into their own section. Pipeline + * definitions used by the LocalTransformServiceRegistry may use node reference options defined in the json + * returned by T-Engines. with the actual definitions from the node options + * reference section. It also combines multiple json sources into a single jsonNode structure that can be parsed as + * before. */ public class CombinedConfig { + private static final String TRANSFORMER_NAME = "transformerName"; private static final String TRANSFORM_CONFIG = "/transform/config"; + private static final String TRANSFORM_OPTIONS = "transformOptions"; + private static final String GROUP = "group"; + private static final String TRANSFORMERS = "transformers"; private final Log log; + private Map allTransformOptions = new HashMap<>(); + private List allTransforms = new ArrayList<>(); + private ObjectMapper jsonObjectMapper = new ObjectMapper(); + private ConfigFileFinder configFileFinder; + private int tEngineCount; - static class TransformAndItsOrigin + static class TransformNodeAndItsOrigin { - final Transformer transformer; + final ObjectNode node; final String baseUrl; final String readFrom; - TransformAndItsOrigin(Transformer transformer, String baseUrl, String readFrom) + TransformNodeAndItsOrigin(ObjectNode node, String baseUrl, String readFrom) { - this.transformer = transformer; + this.node = node; this.baseUrl = baseUrl; this.readFrom = readFrom; } } - Map> combinedTransformOptions = new HashMap<>(); - List combinedTransformers = new ArrayList<>(); + static class TransformAndItsOrigin + { + final InlineTransformer transform; + final String baseUrl; + final String readFrom; - private ObjectMapper jsonObjectMapper = new ObjectMapper(); - private ConfigFileFinder configFileFinder; - private int tEngineCount; + TransformAndItsOrigin(InlineTransformer transform, String baseUrl, String readFrom) + { + this.transform = transform; + this.baseUrl = baseUrl; + this.readFrom = readFrom; + } + } public CombinedConfig(Log log) { @@ -89,21 +111,40 @@ public class CombinedConfig configFileFinder = new ConfigFileFinder(jsonObjectMapper) { @Override - protected void readJson(JsonNode jsonNode, String readFrom, String baseUrl) + protected void readJson(JsonNode jsonNode, String readFromMessage, String baseUrl) throws IOException { - TransformConfig transformConfig = jsonObjectMapper.convertValue(jsonNode, TransformConfig.class); - transformConfig.getTransformOptions().forEach((key, map) -> combinedTransformOptions.put(key, map)); - transformConfig.getTransformers().forEach(transformer -> combinedTransformers.add( - new TransformAndItsOrigin(transformer, baseUrl, readFrom))); + JsonNode transformOptions = jsonNode.get(TRANSFORM_OPTIONS); + if (transformOptions != null && transformOptions.isObject()) + { + Iterator> iterator = transformOptions.fields(); + while (iterator.hasNext()) + { + Map.Entry entry = iterator.next(); + + JsonNode options = entry.getValue(); + if (options.isArray()) + { + String optionsName = entry.getKey(); + allTransformOptions.put(optionsName, (ArrayNode)options); + } + } + } + + JsonNode transformers = jsonNode.get(TRANSFORMERS); + if (transformers != null && transformers.isArray()) + { + for (JsonNode transformer : transformers) + { + if (transformer.isObject()) + { + allTransforms.add(new TransformNodeAndItsOrigin((ObjectNode)transformer, baseUrl, readFromMessage)); + } + } + } } }; } - public boolean addLocalConfig(String path) - { - return configFileFinder.readFiles(path, log); - } - public boolean addRemoteConfig(List urls, String remoteType) { boolean successReadingConfig = true; @@ -149,9 +190,9 @@ public class CombinedConfig try (StringReader reader = new StringReader(content)) { - int transformCount = combinedTransformers.size(); + int transformCount = allTransforms.size(); configFileFinder.readFile(reader, remoteType+" on "+baseUrl, "json", baseUrl, log); - if (transformCount == combinedTransformers.size()) + if (transformCount == allTransforms.size()) { successReadingConfig = false; } @@ -225,14 +266,175 @@ public class CombinedConfig return message; } - public void register(TransformServiceRegistryImpl registry) + public boolean addLocalConfig(String path) throws IOException + { + return configFileFinder.readFiles(path, log); + } + + public void register(TransformServiceRegistryImpl registry) throws IOException { TransformServiceRegistryImpl.Data data = registry.getData(); data.setTEngineCount(tEngineCount); data.setFileCount(configFileFinder.getFileCount()); + List transformers = getTransforms(); + transformers.forEach(t->registry.register(t.transform, t.baseUrl, t.readFrom)); + } - combinedTransformers.forEach(transformer -> - registry.register(transformer.transformer, combinedTransformOptions, - transformer.baseUrl, transformer.readFrom)); + public List getTransforms() throws IOException + { + List transforms = new ArrayList<>(); + + // After all json input has been loaded build the output with the options in place. + ArrayNode transformersNode = jsonObjectMapper.createArrayNode(); + for (TransformNodeAndItsOrigin entity : allTransforms) + { + transformersNode.add(entity.node); + + try + { + ArrayNode transformOptions = (ArrayNode) entity.node.get(TRANSFORM_OPTIONS); + if (transformOptions != null) + { + + ArrayNode options; + int size = transformOptions.size(); + if (size == 1) + { + // If there is a single node option reference, we can just use it. + int i = 0; + options = getTransformOptions(transformOptions, i, entity.node); + } + else + { + // If there are many node option references (typically in a pipeline), then each element + // has a group for each set of node options. + options = jsonObjectMapper.createArrayNode(); + for (int i = size - 1; i >= 0; i--) + { + JsonNode referencedTransformOptions = getTransformOptions(transformOptions, i, entity.node); + if (referencedTransformOptions != null) + { + ObjectNode element = jsonObjectMapper.createObjectNode(); + options.add(element); + + ObjectNode group = jsonObjectMapper.createObjectNode(); + group.set(TRANSFORM_OPTIONS, referencedTransformOptions); + element.set(GROUP, group); + } + } + } + if (options == null || options.size() == 0) + { + entity.node.remove(TRANSFORM_OPTIONS); + } + else + { + entity.node.set(TRANSFORM_OPTIONS, options); + } + } + + try + { + InlineTransformer transform = jsonObjectMapper.convertValue(entity.node, InlineTransformer.class); + transforms.add(new TransformAndItsOrigin(transform, entity.baseUrl, entity.readFrom)); + } + catch (IllegalArgumentException e) + { + log.error("Invalid transformer "+getTransformName(entity.node)+" "+e.getMessage()+" baseUrl="+entity.baseUrl); + } + } + catch (IllegalArgumentException e) + { + String transformString = jsonObjectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(entity.node); + log.error(e.getMessage()); + log.debug(transformString); + } + } + if (log.isTraceEnabled()) + { + log.trace("Combined config:\n"+jsonObjectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(transformersNode)); + } + + transforms = sortTransformers(transforms); + return transforms; + } + + // Sort transformers so there are no forward references, if that is possible. + private List sortTransformers(List original) + { + List transformers = new ArrayList<>(original.size()); + List todo = new ArrayList<>(original.size()); + Set transformerNames = new HashSet<>(); + boolean added; + do + { + added = false; + for (TransformAndItsOrigin entry : original) + { + String name = entry.transform.getTransformerName(); + List pipeline = entry.transform.getTransformerPipeline(); + boolean addEntry = true; + if (pipeline != null && !pipeline.isEmpty()) + { + for (TransformStep step : pipeline) + { + String stepName = step.getTransformerName(); + if (!transformerNames.contains(stepName)) + { + todo.add(entry); + addEntry = false; + break; + } + } + } + if (addEntry) + { + transformers.add(entry); + added = true; + if (name != null) + { + transformerNames.add(name); + } + } + } + + original.clear(); + original.addAll(todo); + todo.clear(); + } + while (added && !original.isEmpty()); + + transformers.addAll(todo); + + return transformers; + } + + private ArrayNode getTransformOptions(ArrayNode transformOptions, int i, ObjectNode transform) + { + ArrayNode options = null; + JsonNode optionName = transformOptions.get(i); + if (optionName.isTextual()) + { + String name = optionName.asText(); + options = allTransformOptions.get(name); + if (options == null) + { + String message = "Reference to \"transformOptions\": \"" + name + "\" not found. Transformer " + + getTransformName(transform) + " ignored."; + throw new IllegalArgumentException(message); + } + } + return options; + } + + private String getTransformName(ObjectNode transform) + { + String name = "Unknown"; + JsonNode nameNode = transform.get(TRANSFORMER_NAME); + if (nameNode != null && nameNode.isTextual()) + { + name = '"'+nameNode.asText()+'"'; + } + return name; } } diff --git a/src/main/java/org/alfresco/transform/client/model/config/TransformServiceRegistry.java b/src/main/java/org/alfresco/transform/client/model/config/TransformServiceRegistry.java new file mode 100644 index 0000000000..c1bcf59f7d --- /dev/null +++ b/src/main/java/org/alfresco/transform/client/model/config/TransformServiceRegistry.java @@ -0,0 +1,67 @@ +/* + * #%L + * Alfresco Repository + * %% + * Copyright (C) 2005 - 2018 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.client.model.config; + +import org.quartz.CronExpression; + +import java.util.Map; + +/** + * Used by clients work out if a transformation is supported by a Transform Service. + */ +public interface TransformServiceRegistry +{ + /** + * Works out if the Transform Server should be able to transform content of a given source mimetype and size into a + * target mimetype given a list of actual transform option names and values (Strings) plus the data contained in the + * {@Transform} objects registered with this class. + * @param sourceMimetype the mimetype of the source content + * @param sourceSizeInBytes the size in bytes of the source content. Ignored if negative. + * @param targetMimetype the mimetype of the target + * @param actualOptions the actual name value pairs available that could be passed to the Transform Service. + * @param transformName (optional) name for the set of options and target mimetype. If supplied is used to cache + * results to avoid having to work out if a given transformation is supported a second time. + * The sourceMimetype and sourceSizeInBytes may still change. In the case of ACS this is the + * rendition name. + */ + boolean isSupported(String sourceMimetype, long sourceSizeInBytes, String targetMimetype, + Map actualOptions, String transformName); + + /** + * Returns the maximun size (in bytes) of the source content that can be transformed. + * @param sourceMimetype the mimetype of the source content + * @param targetMimetype the mimetype of the target + * @param actualOptions the actual name value pairs available that could be passed to the Transform Service. + * @param transformName (optional) name for the set of options and target mimetype. If supplied is used to cache + * results to avoid having to work out if a given transformation is supported a second time. + * The sourceMimetype and sourceSizeInBytes may still change. In the case of ACS this is the + * rendition name. + * @return the maximum size (in bytes) of the source content that can be transformed. If {@code -1} there is no + * limit, but if {@code 0} the transform is not supported. + */ + long getMaxSize(String sourceMimetype, String targetMimetype, + Map actualOptions, String transformName); +} diff --git a/src/main/java/org/alfresco/transform/client/model/config/TransformServiceRegistryImpl.java b/src/main/java/org/alfresco/transform/client/model/config/TransformServiceRegistryImpl.java index 074ac0f53c..56d0fc142c 100644 --- a/src/main/java/org/alfresco/transform/client/model/config/TransformServiceRegistryImpl.java +++ b/src/main/java/org/alfresco/transform/client/model/config/TransformServiceRegistryImpl.java @@ -25,6 +25,7 @@ */ package org.alfresco.transform.client.model.config; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import org.alfresco.util.ConfigScheduler; import org.alfresco.util.PropertyCheck; @@ -33,14 +34,29 @@ import org.quartz.CronExpression; import org.springframework.beans.factory.InitializingBean; import java.io.IOException; +import java.io.Reader; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import static org.alfresco.repo.rendition2.RenditionDefinition2.TIMEOUT; /** * Used by clients to work out if a transformation is supported by the Transform Service. */ -public abstract class TransformServiceRegistryImpl extends AbstractTransformRegistry implements InitializingBean +public abstract class TransformServiceRegistryImpl implements TransformServiceRegistry, InitializingBean { - public static class Data extends AbstractTransformRegistry.Data + public static class Data { + ConcurrentMap>> transformers = new ConcurrentHashMap<>(); + ConcurrentMap>> cachedSupportedTransformList = new ConcurrentHashMap<>(); + private int transformerCount = 0; + private int transformCount = 0; private int tEngineCount = 0; private int fileCount; boolean firstTime = true; @@ -64,6 +80,25 @@ public abstract class TransformServiceRegistryImpl extends AbstractTransformRegi } } + static class SupportedTransform + { + TransformOptionGroup transformOptions; + long maxSourceSizeBytes; + private String name; + private int priority; + + public SupportedTransform(Data data, String name, Set transformOptions, long maxSourceSizeBytes, int priority) + { + // Logically the top level TransformOptionGroup is required, so that child options are optional or required + // based on their own setting. + this.transformOptions = new TransformOptionGroup(true, transformOptions); + this.maxSourceSizeBytes = maxSourceSizeBytes; + this.name = name; + this.priority = priority; + data.transformCount++; + } + } + protected boolean enabled = true; private ObjectMapper jsonObjectMapper; private CronExpression cronExpression; @@ -158,9 +193,267 @@ public abstract class TransformServiceRegistryImpl extends AbstractTransformRegi protected abstract Log getLog(); - @Override - protected void logError(String msg) + public void register(Reader reader, String readFrom) throws IOException { - getLog().error(msg); + List transformers = jsonObjectMapper.readValue(reader, new TypeReference>(){}); + transformers.forEach(t -> register(t, null, readFrom)); + } + + protected void register(InlineTransformer transformer, String baseUrl, String readFrom) + { + Data data = getData(); + data.transformerCount++; + transformer.getSupportedSourceAndTargetList().forEach( + e -> data.transformers.computeIfAbsent(e.getSourceMediaType(), + k -> new ConcurrentHashMap<>()).computeIfAbsent(e.getTargetMediaType(), + k -> new ArrayList<>()).add( + new SupportedTransform(data, transformer.getTransformerName(), + transformer.getTransformOptions(), e.getMaxSourceSizeBytes(), e.getPriority()))); + } + + @Override + public boolean isSupported(String sourceMimetype, long sourceSizeInBytes, String targetMimetype, + Map actualOptions, String renditionName) + { + long maxSize = getMaxSize(sourceMimetype, targetMimetype, actualOptions, renditionName); + return maxSize != 0 && (maxSize == -1L || maxSize >= sourceSizeInBytes); + } + + /** + * Works out the name of the transformer (might not map to an actual transformer) that will be used to transform + * content of a given source mimetype and size into a target mimetype given a list of actual transform option names + * and values (Strings) plus the data contained in the {@Transform} objects registered with this class. + * @param sourceMimetype the mimetype of the source content + * @param sourceSizeInBytes the size in bytes of the source content. Ignored if negative. + * @param targetMimetype the mimetype of the target + * @param actualOptions the actual name value pairs available that could be passed to the Transform Service. + * @param renditionName (optional) name for the set of options and target mimetype. If supplied is used to cache + * results to avoid having to work out if a given transformation is supported a second time. + * The sourceMimetype and sourceSizeInBytes may still change. In the case of ACS this is the + * rendition name. + */ + protected String getTransformerName(String sourceMimetype, long sourceSizeInBytes, String targetMimetype, Map actualOptions, String renditionName) + { + List supportedTransforms = getTransformListBySize(sourceMimetype, targetMimetype, actualOptions, renditionName); + for (SupportedTransform supportedTransform : supportedTransforms) + { + if (supportedTransform.maxSourceSizeBytes == -1 || supportedTransform.maxSourceSizeBytes >= sourceSizeInBytes) + { + return supportedTransform.name; + } + } + + return null; + } + + @Override + public long getMaxSize(String sourceMimetype, String targetMimetype, + Map actualOptions, String renditionName) + { + List supportedTransforms = getTransformListBySize(sourceMimetype, targetMimetype, actualOptions, renditionName); + return supportedTransforms.isEmpty() ? 0 : supportedTransforms.get(supportedTransforms.size()-1).maxSourceSizeBytes; + } + + // Returns transformers in increasing supported size order, where lower priority transformers for the same size have + // been discarded. + private List getTransformListBySize(String sourceMimetype, String targetMimetype, + Map actualOptions, String renditionName) + { + if (actualOptions == null) + { + actualOptions = Collections.EMPTY_MAP; + } + if (renditionName != null && renditionName.trim().isEmpty()) + { + renditionName = null; + } + + Data data = getData(); + List transformListBySize = renditionName == null ? null + : data.cachedSupportedTransformList.computeIfAbsent(renditionName, k -> new ConcurrentHashMap<>()).get(sourceMimetype); + if (transformListBySize != null) + { + return transformListBySize; + } + + // Remove the "timeout" property from the actualOptions as it is not used to select a transformer. + if (actualOptions.containsKey(TIMEOUT)) + { + actualOptions = new HashMap(actualOptions); + actualOptions.remove(TIMEOUT); + } + + transformListBySize = new ArrayList<>(); + ConcurrentMap> targetMap = data.transformers.get(sourceMimetype); + if (targetMap != null) + { + List supportedTransformList = targetMap.get(targetMimetype); + if (supportedTransformList != null) + { + for (SupportedTransform supportedTransform : supportedTransformList) + { + TransformOptionGroup transformOptions = supportedTransform.transformOptions; + Map possibleTransformOptions = new HashMap<>(); + addToPossibleTransformOptions(possibleTransformOptions, transformOptions, true, actualOptions); + if (isSupported(possibleTransformOptions, actualOptions)) + { + addToSupportedTransformList(transformListBySize, supportedTransform); + } + } + } + } + + if (renditionName != null) + { + data.cachedSupportedTransformList.get(renditionName).put(sourceMimetype, transformListBySize); + } + + return transformListBySize; + } + + // Add newTransform to the transformListBySize in increasing size order and discards lower priority (numerically + // higher) transforms with a smaller or equal size. + private void addToSupportedTransformList(List transformListBySize, SupportedTransform newTransform) + { + for (int i=0; i < transformListBySize.size(); i++) + { + SupportedTransform existingTransform = transformListBySize.get(i); + int added = -1; + int compare = compare(newTransform.maxSourceSizeBytes, existingTransform.maxSourceSizeBytes); + if (compare < 0) + { + transformListBySize.add(i, newTransform); + added = i; + } + else if (compare == 0) + { + if (newTransform.priority < existingTransform.priority) + { + transformListBySize.set(i, newTransform); + added = i; + } + } + if (added == i) + { + for (i--; i >= 0; i--) + { + existingTransform = transformListBySize.get(i); + if (newTransform.priority <= existingTransform.priority) + { + transformListBySize.remove(i); + } + } + return; + } + } + transformListBySize.add(newTransform); + } + + // compare where -1 is unlimited. + private int compare(long a, long b) + { + return a == -1 + ? b == -1 ? 0 : 1 + : b == -1 ? -1 + : a == b ? 0 + : a > b ? 1 : -1; + } + + /** + * Flatten out the transform options by adding them to the supplied possibleTransformOptions.

+ * + * If possible discards options in the supplied transformOptionGroup if the group is optional and the actualOptions + * don't provide any of the options in the group. Or to put it another way:

+ * + * It adds individual transform options from the transformOptionGroup to possibleTransformOptions if the group is + * required or if the actualOptions include individual options from the group. As a result it is possible that none + * of the group are added if it is optional. It is also possible to add individual transform options that are + * themselves required but not in the actualOptions. In this the isSupported method will return false. + * @return true if any options were added. Used by nested call parents to determine if an option was added from a + * nested sub group. + */ + boolean addToPossibleTransformOptions(Map possibleTransformOptions, + TransformOptionGroup transformOptionGroup, + Boolean parentGroupRequired, Map actualOptions) + { + boolean added = false; + boolean required = false; + + Set optionList = transformOptionGroup.getTransformOptions(); + if (optionList != null && !optionList.isEmpty()) + { + // We need to avoid adding options from a group that is required but its parents are not. + boolean transformOptionGroupRequired = transformOptionGroup.isRequired() && parentGroupRequired; + + // Check if the group contains options in actualOptions. This will add any options from sub groups. + for (TransformOption transformOption : optionList) + { + if (transformOption instanceof TransformOptionGroup) + { + added = addToPossibleTransformOptions(possibleTransformOptions, (TransformOptionGroup) transformOption, + transformOptionGroupRequired, actualOptions); + required |= added; + } + else + { + String name = ((TransformOptionValue) transformOption).getName(); + if (actualOptions.containsKey(name)) + { + required = true; + } + } + } + + if (required || transformOptionGroupRequired) + { + for (TransformOption transformOption : optionList) + { + if (transformOption instanceof TransformOptionValue) + { + added = true; + TransformOptionValue transformOptionValue = (TransformOptionValue) transformOption; + String name = transformOptionValue.getName(); + boolean optionValueRequired = transformOptionValue.isRequired(); + possibleTransformOptions.put(name, optionValueRequired); + } + } + } + } + + return added; + } + + boolean isSupported(Map transformOptions, Map actualOptions) + { + boolean supported = true; + + // Check all required transformOptions are supplied + for (Map.Entry transformOption : transformOptions.entrySet()) + { + Boolean required = transformOption.getValue(); + if (required) + { + String name = transformOption.getKey(); + if (!actualOptions.containsKey(name)) + { + supported = false; + break; + } + } + } + + if (supported) + { + // Check there are no extra unused actualOptions + for (String actualOption : actualOptions.keySet()) + { + if (!transformOptions.containsKey(actualOption)) + { + supported = false; + break; + } + } + } + return supported; } } diff --git a/src/test/java/org/alfresco/transform/client/model/config/LocalTransformServiceRegistryConfigTest.java b/src/test/java/org/alfresco/transform/client/model/config/LocalTransformServiceRegistryConfigTest.java index 4b910b0f13..b9805c58b3 100644 --- a/src/test/java/org/alfresco/transform/client/model/config/LocalTransformServiceRegistryConfigTest.java +++ b/src/test/java/org/alfresco/transform/client/model/config/LocalTransformServiceRegistryConfigTest.java @@ -43,12 +43,12 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; -import java.util.Set; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * Extends the {@link TransformServiceRegistryConfigTest} (used to test the config received from the Transform Service) @@ -115,7 +115,12 @@ public class LocalTransformServiceRegistryConfigTest extends TransformServiceReg } } - private static Log log = LogFactory.getLog(LocalTransformServiceRegistry.class); + protected TestLocalTransformServiceRegistry registry; + + private Properties properties = new Properties(); + + @Mock private TransformerDebug transformerDebug; + @Mock private MimetypeMap mimetypeMap; private static final String LOCAL_TRANSFORM_SERVICE_CONFIG = "alfresco/local-transform-service-config-test.json"; private static final String LOCAL_TRANSFORM_SERVICE_CONFIG_PIPELINE = "alfresco/local-transform-service-config-pipeline-test.json"; @@ -124,15 +129,7 @@ public class LocalTransformServiceRegistryConfigTest extends TransformServiceReg private static final String LOCAL_TRANSFORM = "localTransform."; private static final String URL = ".url"; - private Map> mapOfTransformOptions; - private List transformerList; - - protected TestLocalTransformServiceRegistry registry; - - private Properties properties = new Properties(); - - @Mock private TransformerDebug transformerDebug; - @Mock private MimetypeMap mimetypeMap; + private static Log log = LogFactory.getLog(LocalTransformServiceRegistry.class); private Map> imagemagickSupportedTransformation; private Map> tikaSupportedTransformation; @@ -190,14 +187,20 @@ public class LocalTransformServiceRegistryConfigTest extends TransformServiceReg } /** - * Loads localTransforms from the LOCAL_TRANSFORM_SERVICE_CONFIG config file. + * Reads and loads localTransforms from LOCAL_TRANSFORM_SERVICE_CONFIG config file. + * @return List list of local transformers. */ - private void retrieveLocalTransformList() + private List retrieveLocalTransformList() { - CombinedConfig combinedConfig = new CombinedConfig(log); - combinedConfig.addLocalConfig(LOCAL_TRANSFORM_SERVICE_CONFIG); - mapOfTransformOptions = combinedConfig.combinedTransformOptions; - transformerList = combinedConfig.combinedTransformers; + try { + CombinedConfig combinedConfig = new CombinedConfig(log); + combinedConfig.addLocalConfig(LOCAL_TRANSFORM_SERVICE_CONFIG); + return combinedConfig.getTransforms(); + } catch (IOException e) { + log.error("Could not read LocalTransform config file"); + fail(); + } + return null; } /** @@ -277,23 +280,11 @@ public class LocalTransformServiceRegistryConfigTest extends TransformServiceReg officeToImageViaPdfSupportedTransformation.put("application/vnd.ms-outlook", targetMimetype); } - protected String getBaseUrl(Transformer transformer) + protected String getBaseUrl(InlineTransformer transformer) { return LOCAL_TRANSFORM+transformer.getTransformerName()+".url"; } - - private int countTopLevelOptions(Set transformOptionNames) - { - int i = 0; - for (String name: transformOptionNames) - { - Set transformOptions = mapOfTransformOptions.get(name); - i += transformOptions.size(); - } - return i; - } - @Test public void testReadWriteJson() throws IOException { @@ -303,8 +294,7 @@ public class LocalTransformServiceRegistryConfigTest extends TransformServiceReg @Test public void testReadJsonConfig() { - retrieveLocalTransformList(); - + List transformerList = retrieveLocalTransformList(); // Assert expected size of the transformers. assertNotNull("Transformer list is null.", transformerList); assertEquals("Unexpected number of transformers retrieved", 5, transformerList.size()); @@ -319,20 +309,18 @@ public class LocalTransformServiceRegistryConfigTest extends TransformServiceReg for (CombinedConfig.TransformAndItsOrigin t : transformerList) { - assertTrue(t.transformer.getTransformerName() + " should be an expected local transformer.", listOfExpectedTransformersName.contains(t.transformer.getTransformerName())); - listOfExpectedTransformersName.remove(t.transformer.getTransformerName()); + assertTrue(t.transform.getTransformerName() + " should be an expected local transformer.", listOfExpectedTransformersName.contains(t.transform.getTransformerName())); + listOfExpectedTransformersName.remove(t.transform.getTransformerName()); - switch (t.transformer.getTransformerName()) + switch (t.transform.getTransformerName()) { case "imagemagick": - assertEquals(t.transformer.getTransformerName() + " incorrect number of supported transform", 14, t.transformer.getSupportedSourceAndTargetList().size()); - assertEquals( t.transformer.getTransformerName() + "incorrect number of transform option names", 1, t.transformer.getTransformOptions().size()); - assertEquals( t.transformer.getTransformerName() + "incorrect number of transform options", 6, countTopLevelOptions(t.transformer.getTransformOptions())); - assertEquals(t.transformer.getTransformerName() + " expected to not be a transformer pipeline", t.transformer.getTransformerPipeline().size(), 0); - assertEquals(t.transformer.getTransformerName() + " expected to not be a failover pipeline", t.transformer.getTransformerFailover().size(), 0); + assertEquals(t.transform.getTransformerName() + " incorrect number of supported transform", 14, t.transform.getSupportedSourceAndTargetList().size()); + assertEquals( t.transform.getTransformerName() + "incorrect number of transform options", 6, t.transform.getTransformOptions().size()); + assertEquals(t.transform.getTransformerName() + " expected to not be a transformer pipeline", t.transform.getTransformerPipeline().size(), 0); //Test supportedSourceAndTargetList - for ( SupportedSourceAndTarget ssat: t.transformer.getSupportedSourceAndTargetList()) + for ( SupportedSourceAndTarget ssat: t.transform.getSupportedSourceAndTargetList()) { assertTrue(ssat.getSourceMediaType() + " not expected to be a supported transform source.", imagemagickSupportedTransformation.containsKey(ssat.getSourceMediaType())); assertTrue(ssat.getTargetMediaType() + " not expected to be a supported transform target for " + ssat.getSourceMediaType(), imagemagickSupportedTransformation.get(ssat.getSourceMediaType()).contains(ssat.getTargetMediaType())); @@ -340,14 +328,12 @@ public class LocalTransformServiceRegistryConfigTest extends TransformServiceReg break; case "tika": - assertEquals(t.transformer.getTransformerName() + " incorrect number of supported transform", 8, t.transformer.getSupportedSourceAndTargetList().size()); - assertEquals( t.transformer.getTransformerName() + "incorrect number of transform option names", 1, t.transformer.getTransformOptions().size()); - assertEquals( t.transformer.getTransformerName() + "incorrect number of transform options", 5, countTopLevelOptions(t.transformer.getTransformOptions())); - assertEquals(t.transformer.getTransformerName() + " expected to not be a transformer pipeline", t.transformer.getTransformerPipeline().size(), 0); - assertEquals(t.transformer.getTransformerName() + " expected to not be a failover pipeline", t.transformer.getTransformerFailover().size(), 0); + assertEquals(t.transform.getTransformerName() + " incorrect number of supported transform", 8, t.transform.getSupportedSourceAndTargetList().size()); + assertEquals( t.transform.getTransformerName() + "incorrect number of transform options", 5, t.transform.getTransformOptions().size()); + assertEquals(t.transform.getTransformerName() + " expected to not be a transformer pipeline", t.transform.getTransformerPipeline().size(), 0); //Test supportedSourceAndTargetList - for ( SupportedSourceAndTarget ssat: t.transformer.getSupportedSourceAndTargetList()) + for ( SupportedSourceAndTarget ssat: t.transform.getSupportedSourceAndTargetList()) { assertTrue(ssat.getSourceMediaType() + " not expected to be a supported transform source.", tikaSupportedTransformation.containsKey(ssat.getSourceMediaType())); assertTrue(ssat.getTargetMediaType() + " not expected to be a supported transform target for " + ssat.getSourceMediaType(), tikaSupportedTransformation.get(ssat.getSourceMediaType()).contains(ssat.getTargetMediaType())); @@ -355,14 +341,12 @@ public class LocalTransformServiceRegistryConfigTest extends TransformServiceReg break; case "pdfrenderer": - assertEquals(t.transformer.getTransformerName() + " incorrect number of supported transform", 1, t.transformer.getSupportedSourceAndTargetList().size()); - assertEquals( t.transformer.getTransformerName() + "incorrect number of transform option names", 1, t.transformer.getTransformOptions().size()); - assertEquals( t.transformer.getTransformerName() + "incorrect number of transform options", 5, countTopLevelOptions(t.transformer.getTransformOptions())); - assertEquals(t.transformer.getTransformerName() + " expected to not be a transformer pipeline", t.transformer.getTransformerPipeline().size(), 0); - assertEquals(t.transformer.getTransformerName() + " expected to not be a failover pipeline", t.transformer.getTransformerFailover().size(), 0); + assertEquals(t.transform.getTransformerName() + " incorrect number of supported transform", 1, t.transform.getSupportedSourceAndTargetList().size()); + assertEquals( t.transform.getTransformerName() + "incorrect number of transform options", 5, t.transform.getTransformOptions().size()); + assertEquals(t.transform.getTransformerName() + " expected to not be a transformer pipeline", t.transform.getTransformerPipeline().size(), 0); //Test supportedSourceAndTargetList - for ( SupportedSourceAndTarget ssat: t.transformer.getSupportedSourceAndTargetList()) + for ( SupportedSourceAndTarget ssat: t.transform.getSupportedSourceAndTargetList()) { assertTrue(ssat.getSourceMediaType() + " not expected to be a supported transform source.", pdfRendererSupportedTransformation.containsKey(ssat.getSourceMediaType())); assertTrue(ssat.getTargetMediaType() + " not expected to be a supported transform target for " + ssat.getSourceMediaType(), pdfRendererSupportedTransformation.get(ssat.getSourceMediaType()).contains(ssat.getTargetMediaType())); @@ -370,14 +354,12 @@ public class LocalTransformServiceRegistryConfigTest extends TransformServiceReg break; case "libreoffice": - assertEquals(t.transformer.getTransformerName() + " incorrect number of supported transform", 9, t.transformer.getSupportedSourceAndTargetList().size()); - assertEquals( t.transformer.getTransformerName() + "incorrect number of transform option names", 0, t.transformer.getTransformOptions().size()); - assertEquals( t.transformer.getTransformerName() + "incorrect number of transform options", 0, countTopLevelOptions(t.transformer.getTransformOptions())); - assertEquals(t.transformer.getTransformerName() + " expected to not be a transformer pipeline", t.transformer.getTransformerPipeline().size(), 0); - assertEquals(t.transformer.getTransformerName() + " expected to not be a failover pipeline", t.transformer.getTransformerFailover().size(), 0); + assertEquals(t.transform.getTransformerName() + " incorrect number of supported transform", 9, t.transform.getSupportedSourceAndTargetList().size()); + assertEquals( t.transform.getTransformerName() + "incorrect number of transform options", t.transform.getTransformOptions().size(), 0); + assertEquals(t.transform.getTransformerName() + " expected to not be a transformer pipeline", t.transform.getTransformerPipeline().size(), 0); //Test supportedSourceAndTargetList - for ( SupportedSourceAndTarget ssat: t.transformer.getSupportedSourceAndTargetList()) + for ( SupportedSourceAndTarget ssat: t.transform.getSupportedSourceAndTargetList()) { assertTrue(ssat.getSourceMediaType() + " not expected to be a supported transform source.", libreofficeSupportedTransformation.containsKey(ssat.getSourceMediaType())); assertTrue(ssat.getTargetMediaType() + " not expected to be a supported transform target for " + ssat.getSourceMediaType(), libreofficeSupportedTransformation.get(ssat.getSourceMediaType()).contains(ssat.getTargetMediaType())); @@ -385,13 +367,12 @@ public class LocalTransformServiceRegistryConfigTest extends TransformServiceReg break; case "officeToImageViaPdf": - assertEquals(t.transformer.getTransformerName() + " incorrect number of supported transform", 28, t.transformer.getSupportedSourceAndTargetList().size()); - assertEquals( t.transformer.getTransformerName() + "incorrect number of transform option names", 2, t.transformer.getTransformOptions().size()); - assertEquals( t.transformer.getTransformerName() + "incorrect number of transform options", 11, countTopLevelOptions(t.transformer.getTransformOptions())); - assertEquals(t.transformer.getTransformerName() + " expected to be a transformer pipeline", t.transformer.getTransformerPipeline().size(), 3); + assertEquals(t.transform.getTransformerName() + " incorrect number of supported transform", 28, t.transform.getSupportedSourceAndTargetList().size()); + assertEquals( t.transform.getTransformerName() + "incorrect number of transform options", 2, t.transform.getTransformOptions().size()); + assertEquals(t.transform.getTransformerName() + " expected to be a transformer pipeline", t.transform.getTransformerPipeline().size(), 3); //Test supportedSourceAndTargetList - for ( SupportedSourceAndTarget ssat: t.transformer.getSupportedSourceAndTargetList()) + for ( SupportedSourceAndTarget ssat: t.transform.getSupportedSourceAndTargetList()) { assertTrue(ssat.getSourceMediaType() + " not expected to be a supported transform source.", officeToImageViaPdfSupportedTransformation.containsKey(ssat.getSourceMediaType())); assertTrue(ssat.getTargetMediaType() + " not expected to be a supported transform target for " + ssat.getSourceMediaType(), officeToImageViaPdfSupportedTransformation.get(ssat.getSourceMediaType()).contains(ssat.getTargetMediaType())); @@ -405,14 +386,13 @@ public class LocalTransformServiceRegistryConfigTest extends TransformServiceReg @Test public void testReadTransformProperties() { - retrieveLocalTransformList(); - + List transformerList = retrieveLocalTransformList(); assertNotNull("Transformer list is null.", transformerList); for (CombinedConfig.TransformAndItsOrigin t : transformerList) { - if(t.transformer.getTransformerPipeline() == null) + if(t.transform.getTransformerPipeline() == null) { - assertNotNull(t.transformer.getTransformerName()+ " JVM property not set.", System.getProperty(LOCAL_TRANSFORM + t.transformer.getTransformerName() + URL)); + assertNotNull(t.transform.getTransformerName()+ " JVM property not set.", System.getProperty(LOCAL_TRANSFORM + t.transform.getTransformerName() + URL)); } } assertEquals("Unexpected pdfrenderer JVM property value", "http://localhost:8090/", System.getProperty(LOCAL_TRANSFORM + "pdfrenderer" + URL)); @@ -422,9 +402,9 @@ public class LocalTransformServiceRegistryConfigTest extends TransformServiceReg for (CombinedConfig.TransformAndItsOrigin t : transformerList) { - if(t.transformer.getTransformerPipeline() == null) + if(t.transform.getTransformerPipeline() == null) { - assertNotNull(t.transformer.getTransformerName()+ " alfresco-global property not set.", properties.getProperty(LOCAL_TRANSFORM + t.transformer.getTransformerName() + URL)); + assertNotNull(t.transform.getTransformerName()+ " alfresco-global property not set.", properties.getProperty(LOCAL_TRANSFORM + t.transform.getTransformerName() + URL)); } } assertEquals("Unexpected pdfrenderer alfresco-global property value", "http://localhost:8090/", properties.getProperty(LOCAL_TRANSFORM + "pdfrenderer" + URL)); diff --git a/src/test/java/org/alfresco/transform/client/model/config/TransformBuilder.java b/src/test/java/org/alfresco/transform/client/model/config/TransformBuilder.java new file mode 100644 index 0000000000..1cd65aed9d --- /dev/null +++ b/src/test/java/org/alfresco/transform/client/model/config/TransformBuilder.java @@ -0,0 +1,54 @@ +/* + * #%L + * Alfresco Repository + * %% + * Copyright (C) 2005 - 2018 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.client.model.config; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Helper class that builds a {@link InlineTransformer} given the source and target types and a pipeline of Transformers + * for creating intermediary content. + */ +public class TransformBuilder +{ + public InlineTransformer buildPipeLine(String name, Set sourceAndTargetList, + List transformerList) + { + Set options = new HashSet<>(transformerList.size()); + transformerList.forEach(t -> + { + // Avoid creating an enpty TransformOptionGroup if the transformer has no options. + // Works with an empty TransformOptionGroup but adds to the complexity. + if (t.getTransformer().getTransformOptions() != null) + { + options.add(new TransformOptionGroup(t.isRequired(), t.getTransformer().getTransformOptions())); + } + }); + return new InlineTransformer(name, options, sourceAndTargetList); + } +} diff --git a/src/test/java/org/alfresco/transform/client/model/config/TransformServiceRegistryConfigTest.java b/src/test/java/org/alfresco/transform/client/model/config/TransformServiceRegistryConfigTest.java index 759ea6be4d..80f0e84c76 100644 --- a/src/test/java/org/alfresco/transform/client/model/config/TransformServiceRegistryConfigTest.java +++ b/src/test/java/org/alfresco/transform/client/model/config/TransformServiceRegistryConfigTest.java @@ -33,15 +33,26 @@ import org.apache.log4j.LogManager; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.quartz.CronExpression; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; import java.io.IOException; +import java.io.Reader; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentMap; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -49,25 +60,36 @@ import static org.junit.Assert.assertTrue; /** * Test the config received from the Transform Service about what it supports. - * - * @author adavis */ -public class TransformServiceRegistryConfigTest extends TransformRegistryTest +public class TransformServiceRegistryConfigTest { private static Log log = LogFactory.getLog(TransformServiceRegistryConfigTest.class); + public static final String GIF = "image/gif"; + public static final String JPEG = "image/jpeg"; public static final String PNG = "image/png"; public static final String TIFF = "image/tiff"; + public static final String PDF = "application/pdf"; + public static final String DOC = "application/msword"; + public static final String XLS = "application/vnd.ms-excel"; + public static final String PPT = "application/vnd.ms-powerpoint"; + public static final String MSG = "application/vnd.ms-outlook"; + public static final String TXT = "text/plain"; private static final String TRANSFORM_SERVICE_CONFIG = "alfresco/transform-service-config-test.json"; private static final String TRANSFORM_SERVICE_CONFIG_PIPELINE = "alfresco/transform-service-config-pipeline-test.json"; public static final ObjectMapper JSON_OBJECT_MAPPER = new ObjectMapper(); + private TransformServiceRegistryImpl registry; + protected TransformBuilder builder; + protected InlineTransformer transformer; + @Before public void setUp() throws Exception { - super.setUp(); + registry = buildTransformServiceRegistryImpl(); + builder = new TransformBuilder(); LogManager.getLogger(TransformServiceRegistryConfigTest.class).setLevel(Level.DEBUG); } @@ -109,11 +131,263 @@ public class TransformServiceRegistryConfigTest extends TransformRegistryTest return TRANSFORM_SERVICE_CONFIG_PIPELINE; } + private void assertAddToPossibleOptions(TransformOptionGroup transformOptionGroup, String actualOptionNames, String expectedNames, String expectedRequired) + { + Map actualOptions = buildActualOptions(actualOptionNames); + Set expectedNameSet = expectedNames == null || expectedNames.isEmpty() ? Collections.EMPTY_SET : new HashSet(Arrays.asList(expectedNames.split(", "))); + Set expectedRequiredSet = expectedRequired == null || expectedRequired.isEmpty() ? Collections.EMPTY_SET : new HashSet(Arrays.asList(expectedRequired.split(", "))); + + Map possibleTransformOptions = new HashMap<>(); + + registry.addToPossibleTransformOptions(possibleTransformOptions, transformOptionGroup, true, actualOptions); + + assertEquals("The expected options don't match", expectedNameSet, possibleTransformOptions.keySet()); + for (String name: possibleTransformOptions.keySet()) + { + Boolean required = possibleTransformOptions.get(name); + if (required) + { + assertTrue(name+" should be REQUIRED", expectedRequiredSet.contains(name)); + } + else + { + assertFalse(name+" should be OPTIONAL", expectedRequiredSet.contains(name)); + } + } + } + + // transformOptionNames are upper case if required. + private void assertIsSupported(String actualOptionNames, String transformOptionNames, String unsupportedMsg) + { + Map actualOptions = buildActualOptions(actualOptionNames); + + Map transformOptions = new HashMap<>(); + Set transformOptionNameSet = transformOptionNames == null || transformOptionNames.isEmpty() ? Collections.EMPTY_SET : new HashSet(Arrays.asList(transformOptionNames.split(", "))); + for (String name : transformOptionNameSet) + { + Boolean required = name.toUpperCase().equals(name); + transformOptions.put(name, required); + } + + boolean supported = registry.isSupported(transformOptions, actualOptions); + if (unsupportedMsg == null || unsupportedMsg.isEmpty()) + { + assertTrue("Expected these options to be SUPPORTED", supported); + } + else + { + assertFalse("Expected these options NOT to be supported, because "+unsupportedMsg, supported); + } + } + + private void assertTransformOptions(Set transformOptions) throws Exception + { + transformer = new InlineTransformer("name", + transformOptions, + Set.of( + new SupportedSourceAndTarget(DOC, TXT, -1), + new SupportedSourceAndTarget(XLS, TXT, 1024000))); + + registry = buildTransformServiceRegistryImpl(); + registry.register(transformer, getBaseUrl(transformer), getClass().getName()); + + assertTrue(registry.isSupported(XLS, 1024, TXT, Collections.emptyMap(), null)); + assertTrue(registry.isSupported(XLS, 1024000, TXT, null, null)); + assertFalse(registry.isSupported(XLS, 1024001, TXT, Collections.emptyMap(), null)); + assertTrue(registry.isSupported(DOC, 1024001, TXT, null, null)); + } + + protected String getBaseUrl(InlineTransformer transformer) + { + return null; + } + + private void assertTransformerName(String sourceMimetype, long sourceSizeInBytes, String targetMimetype, + Map actualOptions, String expectedTransformerName, + InlineTransformer... transformers) throws Exception + { + buildAndPopulateRegistry(transformers); + String transformerName = registry.getTransformerName(sourceMimetype, sourceSizeInBytes, targetMimetype, actualOptions, null); + assertEquals(sourceMimetype+" to "+targetMimetype+" should have returned "+expectedTransformerName, expectedTransformerName, transformerName); + } + + private void assertSupported(String sourceMimetype, long sourceSizeInBytes, String targetMimetype, + Map actualOptions, String unsupportedMsg) throws Exception + { + assertSupported(sourceMimetype, sourceSizeInBytes, targetMimetype, actualOptions, unsupportedMsg, transformer); + } + + private void assertSupported(String sourceMimetype, long sourceSizeInBytes, String targetMimetype, + Map actualOptions, String unsupportedMsg, + InlineTransformer... transformers) throws Exception + { + buildAndPopulateRegistry(transformers); + assertSupported(sourceMimetype, sourceSizeInBytes, targetMimetype, actualOptions, null, unsupportedMsg); + } + + private void buildAndPopulateRegistry(InlineTransformer[] transformers) throws Exception + { + registry = buildTransformServiceRegistryImpl(); + for (InlineTransformer transformer : transformers) + { + registry.register(transformer, getBaseUrl(transformer), getClass().getName()); + } + } + + private void assertSupported(String sourceMimetype, long sourceSizeInBytes, String targetMimetype, + Map actualOptions, String renditionName, + String unsupportedMsg) + { + boolean supported = registry.isSupported(sourceMimetype, sourceSizeInBytes, targetMimetype, actualOptions, renditionName); + if (unsupportedMsg == null || unsupportedMsg.isEmpty()) + { + assertTrue(sourceMimetype+" to "+targetMimetype+" should be SUPPORTED", supported); + } + else + { + assertFalse(sourceMimetype+" to "+targetMimetype+" should NOT be supported", supported); + } + } + + private Map buildActualOptions(String actualOptionNames) + { + Map actualOptions = new HashMap<>(); + Set actualOptionNamesSet = actualOptionNames == null || actualOptionNames.isEmpty() ? Collections.EMPTY_SET : new HashSet(Arrays.asList(actualOptionNames.split(", "))); + for (String name : actualOptionNamesSet) + { + actualOptions.put(name, "value for " + name); + } + return actualOptions; + } + private void register(String path) throws IOException { CombinedConfig combinedConfig = new CombinedConfig(log); combinedConfig.addLocalConfig(path); - combinedConfig.register((TransformServiceRegistryImpl)registry); + combinedConfig.register(registry); + } + + @Test + public void testReadWriteJson() throws IOException + { + InlineTransformer libreoffice = new InlineTransformer("libreoffice", + null, // there are no options + Set.of( + new SupportedSourceAndTarget(DOC, PDF, -1), + new SupportedSourceAndTarget(XLS, PDF, 1024000), + new SupportedSourceAndTarget(PPT, PDF, -1), + new SupportedSourceAndTarget(MSG, PDF, -1))); + + InlineTransformer pdfrenderer = new InlineTransformer("pdfrenderer", + Set.of( + new TransformOptionValue(false, "page"), + new TransformOptionValue(false, "width"), + new TransformOptionValue(false, "height"), + new TransformOptionValue(false, "allowPdfEnlargement"), + new TransformOptionValue(false, "maintainPdfAspectRatio")), + Set.of( + new SupportedSourceAndTarget(PDF, PNG, -1))); + + InlineTransformer tika = new InlineTransformer("tika", + Set.of( + new TransformOptionValue(false, "transform"), + new TransformOptionValue(false, "includeContents"), + new TransformOptionValue(false, "notExtractBookmarksText"), + new TransformOptionValue(false, "targetMimetype"), + new TransformOptionValue(false, "targetEncoding")), + Set.of( + new SupportedSourceAndTarget(PDF, TXT, -1), + new SupportedSourceAndTarget(DOC, TXT, -1), + new SupportedSourceAndTarget(XLS, TXT, 1024000), + new SupportedSourceAndTarget(PPT, TXT, -1), + new SupportedSourceAndTarget(MSG, TXT, -1))); + + InlineTransformer imagemagick = new InlineTransformer("imagemagick", + Set.of( + new TransformOptionValue(false, "alphaRemove"), + new TransformOptionValue(false, "autoOrient"), + new TransformOptionGroup(false, Set.of( + new TransformOptionValue(false, "cropGravity"), + new TransformOptionValue(false, "cropWidth"), + new TransformOptionValue(false, "cropHeight"), + new TransformOptionValue(false, "cropPercentage"), + new TransformOptionValue(false, "cropXOffset"), + new TransformOptionValue(false, "cropYOffset"))), + new TransformOptionGroup(false, Set.of( + new TransformOptionValue(false, "thumbnail"), + new TransformOptionValue(false, "resizeHeight"), + new TransformOptionValue(false, "resizeWidth"), + new TransformOptionValue(false, "resizePercentage"), + new TransformOptionValue(false, "maintainAspectRatio")))), + Set.of( + new SupportedSourceAndTarget(GIF, GIF, -1), + new SupportedSourceAndTarget(GIF, JPEG, -1), + new SupportedSourceAndTarget(GIF, PNG, -1), + new SupportedSourceAndTarget(GIF, TIFF, -1), + + new SupportedSourceAndTarget(JPEG, GIF, -1), + new SupportedSourceAndTarget(JPEG, JPEG, -1), + new SupportedSourceAndTarget(JPEG, PNG, -1), + new SupportedSourceAndTarget(JPEG, TIFF, -1), + + new SupportedSourceAndTarget(PNG, GIF, -1), + new SupportedSourceAndTarget(PNG, JPEG, -1), + new SupportedSourceAndTarget(PNG, PNG, -1), + new SupportedSourceAndTarget(PNG, TIFF, -1), + + new SupportedSourceAndTarget(TIFF, GIF, -1), + new SupportedSourceAndTarget(TIFF, JPEG, -1), + new SupportedSourceAndTarget(TIFF, PNG, -1), + new SupportedSourceAndTarget(TIFF, TIFF, -1))); + + InlineTransformer officeToImage = builder.buildPipeLine("transformer1", + Set.of( + new SupportedSourceAndTarget(DOC, GIF, -1), + new SupportedSourceAndTarget(DOC, JPEG, -1), + new SupportedSourceAndTarget(DOC, PNG, -1), + new SupportedSourceAndTarget(DOC, TIFF, -1), + new SupportedSourceAndTarget(XLS, GIF, -1), + new SupportedSourceAndTarget(XLS, JPEG, -1), + new SupportedSourceAndTarget(XLS, PNG, -1), + new SupportedSourceAndTarget(XLS, TIFF, -1), + new SupportedSourceAndTarget(PPT, GIF, -1), + new SupportedSourceAndTarget(PPT, JPEG, -1), + new SupportedSourceAndTarget(PPT, PNG, -1), + new SupportedSourceAndTarget(PPT, TIFF, -1), + new SupportedSourceAndTarget(MSG, GIF, -1), + new SupportedSourceAndTarget(MSG, JPEG, -1), + new SupportedSourceAndTarget(MSG, PNG, -1), + new SupportedSourceAndTarget(MSG, TIFF, -1)), + Arrays.asList( + new ChildTransformer(false, libreoffice), // to pdf + new ChildTransformer(false, pdfrenderer), // to png + new ChildTransformer(true, imagemagick))); // to other image formats + + List transformers1 = Arrays.asList(libreoffice, tika, pdfrenderer, imagemagick, officeToImage); + + File tempFile = File.createTempFile("test", ".json"); + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.writerWithDefaultPrettyPrinter().writeValue(new FileWriter(tempFile), transformers1); + + try (Reader reader = new BufferedReader(new FileReader(tempFile))) + { + registry.register(reader, getClass().getName()); + // Check the count of transforms supported + assertEquals("The number of UNIQUE source to target mimetypes transforms has changed. Config change?", + 42, countSupportedTransforms(true)); + assertEquals("The number of source to target mimetypes transforms has changed. " + + "There may be multiple transformers for the same combination. Config change?", + 42, countSupportedTransforms(false)); + + // Check a supported transform for each transformer. + assertSupported(DOC, 1234, PDF, null, null, ""); // libreoffice + assertSupported(DOC, 1234, PDF, null, null, ""); // libreoffice + assertSupported(PDF, 1234, PNG, null, null, ""); // pdfrenderer + assertSupported(JPEG,1234, GIF, null, null, ""); // imagemagick + assertSupported(MSG, 1234, TXT, null, null, ""); // tika + assertSupported(MSG, 1234, GIF, null, null, ""); // transformer1 (officeToImageViaPdf) + assertSupported(DOC, 1234, PNG, null, null, ""); // transformer1 (officeToImageViaPdf) + } } @Test @@ -129,12 +403,12 @@ public class TransformServiceRegistryConfigTest extends TransformRegistryTest 60, countSupportedTransforms(false)); // Check a supported transform for each transformer. - assertSupported(DOC, 1234, PDF, actualOptions, null, ""); // libreoffice - assertSupported(DOC, 1234, PDF, actualOptions, null, ""); // libreoffice - assertSupported(PDF, 1234, PNG, actualOptions, null, ""); // pdfrenderer - assertSupported(JPEG,1234, GIF, actualOptions, null, ""); // imagemagick - assertSupported(MSG, 1234, TXT, actualOptions, null, ""); // tika - assertSupported(MSG, 1234, GIF, actualOptions, null, ""); // officeToImageViaPdf + assertSupported(DOC, 1234, PDF, null, null, ""); // libreoffice + assertSupported(DOC, 1234, PDF, null, null, ""); // libreoffice + assertSupported(PDF, 1234, PNG, null, null, ""); // pdfrenderer + assertSupported(JPEG,1234, GIF, null, null, ""); // imagemagick + assertSupported(MSG, 1234, TXT, null, null, ""); // tika + assertSupported(MSG, 1234, GIF, null, null, ""); // officeToImageViaPdf Map invalidPdfOptions = new HashMap<>(); invalidPdfOptions.put("allowEnlargement", "false"); @@ -154,11 +428,14 @@ public class TransformServiceRegistryConfigTest extends TransformRegistryTest "There may be multiple transformers for the same combination. Config change?", expectedTransforms, countSupportedTransforms(false)); + ConcurrentMap> transformer = + registry.getData().transformers.get("officeToImageViaPdf"); + // Check required and optional default correctly - Map> transformsToWord = + ConcurrentMap> transformsToWord = registry.getData().transformers.get(DOC); - List supportedTransforms = transformsToWord.get(GIF); - AbstractTransformRegistry.SupportedTransform supportedTransform = supportedTransforms.get(0); + List supportedTransforms = transformsToWord.get(GIF); + TransformServiceRegistryImpl.SupportedTransform supportedTransform = supportedTransforms.get(0); Set transformOptionsSet = supportedTransform.transformOptions.getTransformOptions(); System.out.println("Nothing"); @@ -202,12 +479,12 @@ public class TransformServiceRegistryConfigTest extends TransformRegistryTest assertFalse("cropWidth should be optional as required is not set", cropWidth.isRequired()); // Check a supported transform for each transformer. - assertSupported(DOC,1234, GIF, actualOptions, null, ""); - assertSupported(DOC,1234, PNG, actualOptions, null, ""); - assertSupported(DOC,1234, JPEG, actualOptions, null, ""); - assertSupported(DOC,1234, TIFF, actualOptions, null, ""); + assertSupported(DOC,1234, GIF, null, null, ""); + assertSupported(DOC,1234, PNG, null, null, ""); + assertSupported(DOC,1234, JPEG, null, null, ""); + assertSupported(DOC,1234, TIFF, null, null, ""); - actualOptions = new HashMap<>(); + Map actualOptions = new HashMap<>(); actualOptions.put("thumbnail", "true"); actualOptions.put("resizeWidth", "100"); actualOptions.put("resizeHeight", "100"); @@ -254,7 +531,16 @@ public class TransformServiceRegistryConfigTest extends TransformRegistryTest private boolean containsTransformOptionValueName (TransformOptionGroup transformOptionGroup, String propertyName) { - return retrieveTransformOptionByPropertyName(transformOptionGroup, propertyName, "TransformOptionValue") != null; + if (retrieveTransformOptionByPropertyName(transformOptionGroup, propertyName, "TransformOptionValue") != null) + return true; + return false; + } + + private boolean containsTransformOptionGroupeName (TransformOptionGroup transformOptionGroup, String propertyName) + { + if (retrieveTransformOptionByPropertyName(transformOptionGroup, propertyName, "TransformOptionGroup") != null) + return true; + return false; } protected int getExpectedTransformsForTestJsonPipeline() @@ -266,9 +552,9 @@ public class TransformServiceRegistryConfigTest extends TransformRegistryTest { int count = 0; int uniqueCount = 0; - for (Map> targetMap : registry.getData().transformers.values()) + for (ConcurrentMap> targetMap : registry.getData().transformers.values()) { - for (List supportedTransforms : targetMap.values()) + for (List supportedTransforms : targetMap.values()) { uniqueCount++; count += supportedTransforms.size(); @@ -276,4 +562,335 @@ public class TransformServiceRegistryConfigTest extends TransformRegistryTest } return unique ? uniqueCount : count; } + + @Test + public void testOptionalGroups() + { + TransformOptionGroup transformOptionGroup = + new TransformOptionGroup(true, Set.of( + new TransformOptionValue(false, "1"), + new TransformOptionValue(true, "2"), + new TransformOptionGroup(false, Set.of( + new TransformOptionValue(false, "3.1"), + new TransformOptionValue(false, "3.2"), + new TransformOptionValue(false, "3.3"))), + new TransformOptionGroup(false, Set.of( // OPTIONAL + new TransformOptionValue(false, "4.1"), + new TransformOptionValue(true, "4.2"), + new TransformOptionValue(false, "4.3"))))); + + assertAddToPossibleOptions(transformOptionGroup, "", "1, 2", "2"); + assertAddToPossibleOptions(transformOptionGroup, "1", "1, 2", "2"); + assertAddToPossibleOptions(transformOptionGroup, "2", "1, 2", "2"); + assertAddToPossibleOptions(transformOptionGroup, "2, 3.2", "1, 2, 3.1, 3.2, 3.3", "2"); + assertAddToPossibleOptions(transformOptionGroup, "2, 4.1", "1, 2, 4.1, 4.2, 4.3", "2, 4.2"); + assertAddToPossibleOptions(transformOptionGroup, "2, 4.2", "1, 2, 4.1, 4.2, 4.3", "2, 4.2"); + } + + @Test + public void testRequiredGroup() + { + TransformOptionGroup transformOptionGroup = + new TransformOptionGroup(true, Set.of( + new TransformOptionValue(false, "1"), + new TransformOptionValue(true, "2"), + new TransformOptionGroup(false, Set.of( + new TransformOptionValue(false, "3.1"), + new TransformOptionValue(false, "3.2"), + new TransformOptionValue(false, "3.3"))), + new TransformOptionGroup(true, Set.of( // REQUIRED + new TransformOptionValue(false, "4.1"), + new TransformOptionValue(true, "4.2"), + new TransformOptionValue(false, "4.3"))))); + + assertAddToPossibleOptions(transformOptionGroup, "", "1, 2, 4.1, 4.2, 4.3", "2, 4.2"); + assertAddToPossibleOptions(transformOptionGroup, "1", "1, 2, 4.1, 4.2, 4.3", "2, 4.2"); + assertAddToPossibleOptions(transformOptionGroup, "2, 3.2", "1, 2, 3.1, 3.2, 3.3, 4.1, 4.2, 4.3", "2, 4.2"); + assertAddToPossibleOptions(transformOptionGroup, "2, 4.1", "1, 2, 4.1, 4.2, 4.3", "2, 4.2"); + assertAddToPossibleOptions(transformOptionGroup, "2, 4.2", "1, 2, 4.1, 4.2, 4.3", "2, 4.2"); + } + + @Test + public void testNesstedGrpups() + { + TransformOptionGroup transformOptionGroup = + new TransformOptionGroup(false, Set.of( + new TransformOptionGroup(false, Set.of( + new TransformOptionValue(false, "1"), + new TransformOptionGroup(false, Set.of( + new TransformOptionValue(false, "1.2"), + new TransformOptionGroup(false, Set.of( + new TransformOptionValue(false, "1.2.3"))))))), + new TransformOptionGroup(false, Set.of( + new TransformOptionValue(false, "2"), + new TransformOptionGroup(false, Set.of( + new TransformOptionValue(false, "2.2"), + new TransformOptionGroup(false, Set.of( + new TransformOptionGroup(false, Set.of( + new TransformOptionValue(false, "2.2.1.2"))))))))), + new TransformOptionGroup(false, Set.of( + new TransformOptionValue(true, "3"), // REQUIRED + new TransformOptionGroup(false, Set.of( + new TransformOptionGroup(false, Set.of( + new TransformOptionGroup(false, Set.of( + new TransformOptionValue(false, "3.1.1.2"))))))))), + new TransformOptionGroup(false, Set.of( + new TransformOptionValue(false, "4"), + new TransformOptionGroup(true, Set.of( // REQUIRED + new TransformOptionGroup(false, Set.of( + new TransformOptionGroup(false, Set.of( + new TransformOptionValue(false, "4.1.1.2"))))))))), + new TransformOptionGroup(false, Set.of( + new TransformOptionValue(false, "5"), + new TransformOptionGroup(false, Set.of( + new TransformOptionGroup(true, Set.of( // REQUIRED + new TransformOptionGroup(false, Set.of( + new TransformOptionValue(false, "5.1.1.2"))))))))), + new TransformOptionGroup(false, Set.of( + new TransformOptionValue(false, "6"), + new TransformOptionGroup(false, Set.of( + new TransformOptionGroup(false, Set.of( + new TransformOptionGroup(true, Set.of( // REQUIRED + new TransformOptionValue(false, "6.1.1.2"))))))))), + new TransformOptionGroup(false, Set.of( + new TransformOptionValue(false, "7"), + new TransformOptionGroup(false, Set.of( + new TransformOptionGroup(false, Set.of( + new TransformOptionGroup(false, Set.of( + new TransformOptionValue(true, "7.1.1.2"))))))))) // REQUIRED + )); + + assertAddToPossibleOptions(transformOptionGroup, "", "", ""); + assertAddToPossibleOptions(transformOptionGroup, "1", "1", ""); + assertAddToPossibleOptions(transformOptionGroup, "1, 7", "1, 7", ""); + assertAddToPossibleOptions(transformOptionGroup, "1, 7.1.1.2", "1, 7, 7.1.1.2", "7.1.1.2"); + assertAddToPossibleOptions(transformOptionGroup, "1, 6", "1, 6", ""); + assertAddToPossibleOptions(transformOptionGroup, "1, 6.1.1.2", "1, 6, 6.1.1.2", ""); + assertAddToPossibleOptions(transformOptionGroup, "1, 5", "1, 5", ""); + assertAddToPossibleOptions(transformOptionGroup, "1, 5.1.1.2", "1, 5, 5.1.1.2", ""); + assertAddToPossibleOptions(transformOptionGroup, "1, 4", "1, 4", ""); + assertAddToPossibleOptions(transformOptionGroup, "1, 4.1.1.2", "1, 4, 4.1.1.2", ""); + assertAddToPossibleOptions(transformOptionGroup, "1, 3", "1, 3", "3"); + assertAddToPossibleOptions(transformOptionGroup, "1, 3.1.1.2", "1, 3, 3.1.1.2", "3"); + + assertAddToPossibleOptions(transformOptionGroup, "2", "2", ""); + assertAddToPossibleOptions(transformOptionGroup, "2, 2.2", "2, 2.2", ""); + assertAddToPossibleOptions(transformOptionGroup, "3", "3", "3"); + assertAddToPossibleOptions(transformOptionGroup, "3.1.1.2", "3, 3.1.1.2", "3"); + } + + @Test + public void testSupportedOptions() + { + assertIsSupported("a", "a, B, c", "required option B is missing"); + assertIsSupported("", "a, B, c", "required option B is missing"); + assertIsSupported("B", "a, B, c", null); + assertIsSupported("B, c", "a, B, c", null); + assertIsSupported("B, a, c", "a, B, c", null); + + assertIsSupported("B, d", "a, B, c", "there is an extra option d"); + assertIsSupported("B, c, d", "a, B, c", "there is an extra option d"); + assertIsSupported("d", "a, B, c", "required option B is missing and there is an extra option d"); + + assertIsSupported("a", "a, b, c", null); + assertIsSupported("", "a, b, c", null); + assertIsSupported("a, b, c", "a, b, c", null); + } + + @Test + public void testNoActualOptions() throws Exception + { + assertTransformOptions(Set.of( + new TransformOptionValue(false, "option1"), + new TransformOptionValue(false, "option2"))); + } + + @Test + public void testNoTrasformOptions() throws Exception + { + assertTransformOptions(Collections.emptySet()); + assertTransformOptions(null); + } + + @Test + public void testSupported() throws Exception + { + transformer = new InlineTransformer("name", + Set.of( + new TransformOptionValue(false, "page"), + new TransformOptionValue(false, "width"), + new TransformOptionValue(false, "height")), + Set.of( + new SupportedSourceAndTarget(DOC, GIF, 102400), + new SupportedSourceAndTarget(DOC, JPEG, -1), + new SupportedSourceAndTarget(MSG, GIF, -1))); + + assertSupported(DOC, 1024, GIF, null, null); + assertSupported(DOC, 102400, GIF, null, null); + assertSupported(DOC, 102401, GIF, null, "source is too large"); + assertSupported(DOC, 1024, JPEG, null, null); + assertSupported(GIF, 1024, DOC, null, GIF+" is not a source of this transformer"); + assertSupported(MSG, 1024, GIF, null, null); + assertSupported(MSG, 1024, JPEG, null, MSG+" to "+JPEG+" is not supported by this transformer"); + + assertSupported(DOC, 1024, GIF, buildActualOptions("page, width"), null); + assertSupported(DOC, 1024, GIF, buildActualOptions("page, width, startPage"), "startPage is not an option"); + } + + @Test + public void testCache() + { + // Note: transformNames are an alias for a set of actualOptions and the target mimetpe. The source mimetype may change. + transformer = new InlineTransformer("name", + Set.of( + new TransformOptionValue(false, "page"), + new TransformOptionValue(false, "width"), + new TransformOptionValue(false, "height")), + Set.of( + new SupportedSourceAndTarget(DOC, GIF, 102400), + new SupportedSourceAndTarget(MSG, GIF, -1))); + + registry.register(transformer, getBaseUrl(transformer), getClass().getName()); + + assertSupported(DOC, 1024, GIF, null, "doclib", ""); + assertSupported(MSG, 1024, GIF, null, "doclib", ""); + + assertEquals(102400L, registry.getMaxSize(DOC, GIF, null, "doclib")); + assertEquals(-1L, registry.getMaxSize(MSG, GIF, null, "doclib")); + + // Change the cached value and try and check we are now using the cached value. + List supportedTransforms = registry.getData().cachedSupportedTransformList.get("doclib").get(DOC); + supportedTransforms.get(0).maxSourceSizeBytes = 1234L; + assertEquals(1234L, registry.getMaxSize(DOC, GIF, null, "doclib")); + } + + @Test + public void testGetTransformerName() throws Exception + { + InlineTransformer t1 = new InlineTransformer("transformer1", null, + Set.of(new SupportedSourceAndTarget(MSG, GIF, 100, 50))); + InlineTransformer t2 = new InlineTransformer("transformer2", null, + Set.of(new SupportedSourceAndTarget(MSG, GIF, 200, 60))); + InlineTransformer t3 = new InlineTransformer("transformer3", null, + Set.of(new SupportedSourceAndTarget(MSG, GIF, 200, 40))); + InlineTransformer t4 = new InlineTransformer("transformer4", null, + Set.of(new SupportedSourceAndTarget(MSG, GIF, -1, 100))); + InlineTransformer t5 = new InlineTransformer("transformer5", null, + Set.of(new SupportedSourceAndTarget(MSG, GIF, -1, 80))); + + Map actualOptions = null; + + // Select on size - priority is ignored + assertTransformerName(MSG, 100, GIF, actualOptions, "transformer1", t1, t2); + assertTransformerName(MSG, 150, GIF, actualOptions, "transformer2", t1, t2); + assertTransformerName(MSG, 250, GIF, actualOptions, null, t1, t2); + // Select on priority - t1, t2 and t4 are discarded. + // t3 is a higher priority and has a larger size than t1 and t2. + // Similar story fo t4 with t5. + assertTransformerName(MSG, 100, GIF, actualOptions, "transformer3", t1, t2, t3, t4, t5); + assertTransformerName(MSG, 200, GIF, actualOptions, "transformer3", t1, t2, t3, t4, t5); + // Select on size and priority, t1 and t2 discarded + assertTransformerName(MSG, 200, GIF, actualOptions, "transformer3", t1, t2, t3, t4); + assertTransformerName(MSG, 300, GIF, actualOptions, "transformer4", t1, t2, t3, t4); + assertTransformerName(MSG, 300, GIF, actualOptions, "transformer5", t1, t2, t3, t4, t5); + } + + @Test + public void testMultipleTransformers() throws Exception + { + InlineTransformer transformer1 = new InlineTransformer("transformer1", + Set.of( + new TransformOptionValue(false, "page"), + new TransformOptionValue(false, "width"), + new TransformOptionValue(false, "height")), + Set.of( + new SupportedSourceAndTarget(DOC, GIF, 102400), + new SupportedSourceAndTarget(DOC, JPEG, -1), + new SupportedSourceAndTarget(MSG, GIF, -1))); + + InlineTransformer transformer2 = new InlineTransformer("transformer2", + Set.of( + new TransformOptionValue(false, "opt1"), + new TransformOptionValue(false, "opt2")), + Set.of( + new SupportedSourceAndTarget(PDF, GIF, -1), + new SupportedSourceAndTarget(PPT, JPEG, -1))); + + InlineTransformer transformer3 = new InlineTransformer("transformer3", + Set.of( + new TransformOptionValue(false, "opt1")), + Set.of( + new SupportedSourceAndTarget(DOC, GIF, -1))); + + Map actualOptions = null; + + assertSupported(DOC, 1024, GIF, actualOptions, null, transformer1); + assertSupported(DOC, 1024, GIF, actualOptions, null, transformer1, transformer2); + assertSupported(DOC, 1024, GIF, actualOptions, null, transformer1, transformer2, transformer3); + + assertSupported(DOC, 102401, GIF, null, "source is too large", transformer1); + assertSupported(DOC, 102401, GIF, null, null, transformer1, transformer3); + + assertSupported(PDF, 1024, GIF, actualOptions, "Only transformer2 supports these mimetypes", transformer1); + assertSupported(PDF, 1024, GIF, actualOptions, null, transformer1, transformer2); + assertSupported(PDF, 1024, GIF, actualOptions, null, transformer1, transformer2, transformer3); + + actualOptions = buildActualOptions("opt1"); + assertSupported(PDF, 1024, GIF, actualOptions, "Only transformer2/4 supports these options", transformer1); + assertSupported(PDF, 1024, GIF, actualOptions, null, transformer1, transformer2); + assertSupported(PDF, 1024, GIF, actualOptions, null, transformer1, transformer2, transformer3); + assertSupported(PDF, 1024, GIF, actualOptions, "transformer4 supports opt1 but not the source mimetype ", transformer1, transformer3); + } + + @Test + public void testPipeline() throws Exception + { + InlineTransformer transformer1 = new InlineTransformer("transformer1", + null, // there are no options + Set.of( + new SupportedSourceAndTarget(DOC, PDF, -1), + new SupportedSourceAndTarget(MSG, PDF, -1))); + + InlineTransformer transformer2 = new InlineTransformer("transformer2", + Set.of( + new TransformOptionValue(false, "page"), + new TransformOptionValue(false, "width"), + new TransformOptionValue(false, "height")), + Set.of( + new SupportedSourceAndTarget(PDF, GIF, -1), + new SupportedSourceAndTarget(PDF, JPEG, -1))); + + buildPipelineTransformer(transformer1, transformer2); + + assertSupported(DOC, 1024, GIF, null, null); + assertSupported(DOC, 1024, JPEG, null, null); + assertSupported(GIF, 1024, DOC, null, GIF+" is not a source of this transformer"); + assertSupported(MSG, 1024, GIF, null, null); + assertSupported(MSG, 1024, JPEG, null, MSG+" to "+JPEG+" is not supported by this transformer"); + + // Now try the options + assertSupported(DOC, 1024, GIF, buildActualOptions("page, width"), null); + assertSupported(DOC, 1024, GIF, buildActualOptions("page, width, startPage"), "startPage is not an option"); + + // Add options to the first transformer + transformer1.setTransformOptions(Set.of( + new TransformOptionValue(false, "startPage"), + new TransformOptionValue(false, "endPage"))); + buildPipelineTransformer(transformer1, transformer2); + + assertSupported(DOC, 1024, GIF, buildActualOptions("page, width"), null); + assertSupported(DOC, 1024, GIF, buildActualOptions("page, width, startPage"), null); + } + + private void buildPipelineTransformer(InlineTransformer transformer1, InlineTransformer transformer2) + { + transformer = builder.buildPipeLine("transformer1", + Set.of( + new SupportedSourceAndTarget(DOC, GIF, -1), + new SupportedSourceAndTarget(DOC, JPEG, -1), + new SupportedSourceAndTarget(MSG, GIF, -1)), + Arrays.asList( + new ChildTransformer(false, transformer1), + new ChildTransformer(true, transformer2))); + } } \ No newline at end of file