From 13ba2534ea8c8514faa39aeac3a5e09c63db5c34 Mon Sep 17 00:00:00 2001 From: alandavis Date: Sat, 13 Aug 2022 18:38:08 +0100 Subject: [PATCH] Allow T-Router to be a T-Engine --- .../config/WebApplicationConfig.java | 2 +- .../transform/coreaio/AIOTransformEngine.java | 4 +- engines/base/README.md | 2 +- engines/base/pom.xml | 5 - .../transform/base/TransformController.java | 4 +- .../transform/base/TransformEngine.java | 2 +- .../base/config/WebApplicationConfig.java | 9 +- .../base/messaging/MessagingConfig.java | 11 +- .../base/messaging/MessagingErrorHandler.java | 31 ++ .../base/messaging/TransformReplySender.java | 3 +- .../base/registry/TransformConfigFiles.java | 15 +- .../registry/TransformConfigFromFiles.java | 2 +- .../base/registry/TransformRegistry.java | 164 +++++++++- .../base/transform/ProcessHandler.java | 32 +- .../base/src/main/resources/application.yaml | 31 +- .../base/TransformControllerAllInOneTest.java | 2 +- .../base/TransformControllerTest.java | 2 +- .../TransformRegistryRefreshTest.java | 2 +- .../base/registry/TransformRegistryTest.java | 56 +++- .../example/HelloTransformEngine.java | 4 +- .../ImageMagickTransformEngine.java | 4 +- .../LibreOfficeTransformEngine.java | 4 +- engines/misc/pom.xml | 2 +- .../transform/misc/MiscTransformEngine.java | 4 +- .../PdfRendererTransformEngine.java | 4 +- .../transform/tika/TikaTransformEngine.java | 4 +- model/pom.xml | 4 + .../transform/common/TransformerDebug.java | 16 +- .../transform/config/TransformConfig.java | 29 +- .../config/TransformOptionGroup.java | 2 +- .../transform/config/Transformer.java | 8 +- .../config/reader/TransformConfigReader.java | 31 ++ .../reader/TransformConfigReaderFactory.java | 53 ++++ .../reader/TransformConfigReaderJson.java | 47 +++ .../reader/TransformConfigReaderYaml.java | 48 +++ .../TransformConfigResourceReader.java | 20 +- .../registry/CombinedTransformConfig.java | 6 + .../registry/TransformRegistryHelper.java | 52 +-- .../common/TransformerDebugTest.java | 48 ++- .../reader/TransformConfigReaderJsonTest.java | 300 ++++++++++++++++++ .../reader/TransformConfigReaderYamlTest.java | 245 ++++++++++++++ .../registry/TransformRegistryHelperTest.java | 18 +- model/src/test/resources/config/sample1.json | 3 + model/src/test/resources/config/sample2.json | 92 ++++++ model/src/test/resources/config/sample3.yaml | 2 + model/src/test/resources/config/sample4.yaml | 130 ++++++++ model/src/test/resources/config/sample5.json | 37 +++ pom.xml | 1 - 48 files changed, 1425 insertions(+), 172 deletions(-) create mode 100644 engines/base/src/main/java/org/alfresco/transform/base/messaging/MessagingErrorHandler.java create mode 100644 model/src/main/java/org/alfresco/transform/config/reader/TransformConfigReader.java create mode 100644 model/src/main/java/org/alfresco/transform/config/reader/TransformConfigReaderFactory.java create mode 100644 model/src/main/java/org/alfresco/transform/config/reader/TransformConfigReaderJson.java create mode 100644 model/src/main/java/org/alfresco/transform/config/reader/TransformConfigReaderYaml.java rename model/src/main/java/org/alfresco/transform/{common => config/reader}/TransformConfigResourceReader.java (72%) create mode 100644 model/src/test/java/org/alfresco/transform/config/reader/TransformConfigReaderJsonTest.java create mode 100644 model/src/test/java/org/alfresco/transform/config/reader/TransformConfigReaderYamlTest.java create mode 100644 model/src/test/resources/config/sample1.json create mode 100644 model/src/test/resources/config/sample2.json create mode 100644 model/src/test/resources/config/sample3.yaml create mode 100644 model/src/test/resources/config/sample4.yaml create mode 100644 model/src/test/resources/config/sample5.json diff --git a/deprecated/alfresco-transformer-base/src/main/java/org/alfresco/transformer/config/WebApplicationConfig.java b/deprecated/alfresco-transformer-base/src/main/java/org/alfresco/transformer/config/WebApplicationConfig.java index d6e10f59..b34e2384 100644 --- a/deprecated/alfresco-transformer-base/src/main/java/org/alfresco/transformer/config/WebApplicationConfig.java +++ b/deprecated/alfresco-transformer-base/src/main/java/org/alfresco/transformer/config/WebApplicationConfig.java @@ -88,6 +88,6 @@ public class WebApplicationConfig implements WebMvcConfigurer @Bean public TransformerDebug transformerDebug() { - return new TransformerDebug().setIsTEngine(true); + return new TransformerDebug().setIsTRouter(false); } } diff --git a/engines/aio/src/main/java/org/alfresco/transform/coreaio/AIOTransformEngine.java b/engines/aio/src/main/java/org/alfresco/transform/coreaio/AIOTransformEngine.java index 5f77d6f9..1ce4864a 100644 --- a/engines/aio/src/main/java/org/alfresco/transform/coreaio/AIOTransformEngine.java +++ b/engines/aio/src/main/java/org/alfresco/transform/coreaio/AIOTransformEngine.java @@ -49,7 +49,7 @@ public class AIOTransformEngine implements TransformEngine @Override public String getTransformEngineName() { - return "0060-AllInOne"; + return "0060 AllInOne"; } @Override @@ -77,7 +77,7 @@ public class AIOTransformEngine implements TransformEngine @Override public ProbeTransform getProbeTransform() { - return new ProbeTransform("quick.pdf", MIMETYPE_PDF, MIMETYPE_TEXT_PLAIN, Collections.emptyMap(), + return new ProbeTransform("probe.pdf", MIMETYPE_PDF, MIMETYPE_TEXT_PLAIN, Collections.emptyMap(), 60, 16, 400, 10240, 60 * 30 + 1, 60 * 15 + 20); } } diff --git a/engines/base/README.md b/engines/base/README.md index e1c554ce..6012ce68 100644 --- a/engines/base/README.md +++ b/engines/base/README.md @@ -41,7 +41,7 @@ package org.alfresco.transform.example; import com.google.common.collect.ImmutableMap; import org.alfresco.transform.base.TransformEngine; import org.alfresco.transform.base.probes.ProbeTransform; -import org.alfresco.transform.common.TransformConfigResourceReader; +import org.alfresco.transform.config.reader.TransformConfigResourceReader; import org.alfresco.transform.config.TransformConfig; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; diff --git a/engines/base/pom.xml b/engines/base/pom.xml index 044d38b7..f885916f 100644 --- a/engines/base/pom.xml +++ b/engines/base/pom.xml @@ -66,10 +66,6 @@ org.springframework.boot spring-boot-starter-activemq - - org.apache.activemq - activemq-client - com.fasterxml.jackson.core jackson-annotations @@ -86,7 +82,6 @@ ch.qos.logback logback-classic - 1.2.6 org.apache.httpcomponents diff --git a/engines/base/src/main/java/org/alfresco/transform/base/TransformController.java b/engines/base/src/main/java/org/alfresco/transform/base/TransformController.java index 69ed2f1f..1ed2896f 100644 --- a/engines/base/src/main/java/org/alfresco/transform/base/TransformController.java +++ b/engines/base/src/main/java/org/alfresco/transform/base/TransformController.java @@ -162,7 +162,7 @@ public class TransformController @ResponseBody public String version() { - return transformEngine.getTransformEngineName() + ' ' + coreVersion + " available"; + return transformEngine.getTransformEngineName() + ' ' + coreVersion; } /** @@ -211,7 +211,7 @@ public class TransformController String pathPrefix = ""; if (behindIngres) { - int i = transformEngineName.lastIndexOf('-'); + int i = transformEngineName.indexOf(' '); if (i != -1) { transformEngineName = transformEngineName.substring(i + 1); diff --git a/engines/base/src/main/java/org/alfresco/transform/base/TransformEngine.java b/engines/base/src/main/java/org/alfresco/transform/base/TransformEngine.java index 71a01c6c..07d6ea06 100644 --- a/engines/base/src/main/java/org/alfresco/transform/base/TransformEngine.java +++ b/engines/base/src/main/java/org/alfresco/transform/base/TransformEngine.java @@ -27,7 +27,7 @@ package org.alfresco.transform.base; import org.alfresco.transform.base.probes.ProbeTransform; -import org.alfresco.transform.common.TransformConfigResourceReader; +import org.alfresco.transform.config.reader.TransformConfigResourceReader; import org.alfresco.transform.config.TransformConfig; /** diff --git a/engines/base/src/main/java/org/alfresco/transform/base/config/WebApplicationConfig.java b/engines/base/src/main/java/org/alfresco/transform/base/config/WebApplicationConfig.java index d8650c63..4ffb51e3 100644 --- a/engines/base/src/main/java/org/alfresco/transform/base/config/WebApplicationConfig.java +++ b/engines/base/src/main/java/org/alfresco/transform/base/config/WebApplicationConfig.java @@ -26,14 +26,10 @@ */ package org.alfresco.transform.base.config; -import org.alfresco.transform.base.clients.AlfrescoSharedFileStoreClient; import org.alfresco.transform.base.html.TransformInterceptor; import org.alfresco.transform.base.registry.TransformConfigSource; -import org.alfresco.transform.base.registry.TransformRegistry; import org.alfresco.transform.common.TransformerDebug; import org.alfresco.transform.messages.TransformRequestValidator; -import org.alfresco.transform.registry.TransformServiceRegistry; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; @@ -58,6 +54,9 @@ public class WebApplicationConfig implements WebMvcConfigurer @Value("${transform.core.version}") private String coreVersionString; + @Value("${container.isTRouter}") + private boolean isTRouter; + @Override public void addInterceptors(InterceptorRegistry registry) { @@ -80,7 +79,7 @@ public class WebApplicationConfig implements WebMvcConfigurer @Bean public TransformerDebug transformerDebug() { - return new TransformerDebug().setIsTEngine(true); + return new TransformerDebug().setIsTRouter(isTRouter); } @Bean diff --git a/engines/base/src/main/java/org/alfresco/transform/base/messaging/MessagingConfig.java b/engines/base/src/main/java/org/alfresco/transform/base/messaging/MessagingConfig.java index 4f386e1f..276f942a 100644 --- a/engines/base/src/main/java/org/alfresco/transform/base/messaging/MessagingConfig.java +++ b/engines/base/src/main/java/org/alfresco/transform/base/messaging/MessagingConfig.java @@ -56,8 +56,6 @@ import org.springframework.transaction.PlatformTransactionManager; @ConditionalOnProperty(name = "activemq.url") public class MessagingConfig implements JmsListenerConfigurer { - private static final Logger logger = LoggerFactory.getLogger(MessagingConfig.class); - @Override public void configureJmsListeners(@NonNull JmsListenerEndpointRegistrar registrar) { @@ -65,7 +63,6 @@ public class MessagingConfig implements JmsListenerConfigurer } @Bean - @ConditionalOnProperty(name = "activemq.url") public DefaultMessageHandlerMethodFactory methodFactory() { DefaultMessageHandlerMethodFactory factory = new DefaultMessageHandlerMethodFactory(); @@ -74,21 +71,20 @@ public class MessagingConfig implements JmsListenerConfigurer } @Bean - @ConditionalOnProperty(name = "activemq.url") public DefaultJmsListenerContainerFactory jmsListenerContainerFactory( final ConnectionFactory connectionFactory, - final TransformMessageConverter transformMessageConverter) + final TransformMessageConverter transformMessageConverter, + final MessagingErrorHandler messagingErrorHandler) { final DefaultJmsListenerContainerFactory factory = new DefaultJmsListenerContainerFactory(); factory.setConnectionFactory(connectionFactory); factory.setMessageConverter(transformMessageConverter); - factory.setErrorHandler(t -> logger.error("JMS error: " + t.getMessage(), t)); + factory.setErrorHandler(messagingErrorHandler); factory.setTransactionManager(transactionManager(connectionFactory)); return factory; } @Bean - @ConditionalOnProperty(name = "activemq.url") public PlatformTransactionManager transactionManager(final ConnectionFactory connectionFactory) { final JmsTransactionManager transactionManager = new JmsTransactionManager(); @@ -97,7 +93,6 @@ public class MessagingConfig implements JmsListenerConfigurer } @Bean - @ConditionalOnProperty(name = "activemq.url") public Queue engineRequestQueue( @Value("${queue.engineRequestQueue}") String engineRequestQueueValue) { diff --git a/engines/base/src/main/java/org/alfresco/transform/base/messaging/MessagingErrorHandler.java b/engines/base/src/main/java/org/alfresco/transform/base/messaging/MessagingErrorHandler.java new file mode 100644 index 00000000..93d116cc --- /dev/null +++ b/engines/base/src/main/java/org/alfresco/transform/base/messaging/MessagingErrorHandler.java @@ -0,0 +1,31 @@ +/* + * Copyright 2015-2018 Alfresco Software, Ltd. All rights reserved. + * + * License rights for this program may be obtained from Alfresco Software, Ltd. + * pursuant to a written agreement and any use of this program without such an + * agreement is prohibited. + */ + +package org.alfresco.transform.base.messaging; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import org.springframework.util.ErrorHandler; + +/** + * Extensible Error Handler for JMS exceptions + * + * @author Cezar Leahu + */ +@Service +public class MessagingErrorHandler implements ErrorHandler +{ + private static final Logger logger = LoggerFactory.getLogger(MessagingErrorHandler.class); + + @Override + public void handleError(Throwable t) + { + logger.error("JMS error: " + t.getMessage(), t); + } +} diff --git a/engines/base/src/main/java/org/alfresco/transform/base/messaging/TransformReplySender.java b/engines/base/src/main/java/org/alfresco/transform/base/messaging/TransformReplySender.java index c3a37ee8..32eb82e4 100644 --- a/engines/base/src/main/java/org/alfresco/transform/base/messaging/TransformReplySender.java +++ b/engines/base/src/main/java/org/alfresco/transform/base/messaging/TransformReplySender.java @@ -36,8 +36,7 @@ import org.springframework.jms.core.JmsTemplate; import org.springframework.stereotype.Component; /** - * Copied from the t-router. We would need to create a common dependency between t-engine base and t-router that - * knows about jms to remove this duplication. + * Copied from the t-router. * * @author Cezar Leahu */ diff --git a/engines/base/src/main/java/org/alfresco/transform/base/registry/TransformConfigFiles.java b/engines/base/src/main/java/org/alfresco/transform/base/registry/TransformConfigFiles.java index 9c3a7442..fd7e63a1 100644 --- a/engines/base/src/main/java/org/alfresco/transform/base/registry/TransformConfigFiles.java +++ b/engines/base/src/main/java/org/alfresco/transform/base/registry/TransformConfigFiles.java @@ -35,15 +35,20 @@ import java.util.List; import java.util.Map; @Configuration -@ConfigurationProperties(prefix = "transform") +@ConfigurationProperties(prefix = "transform.config") public class TransformConfigFiles { - // Populated from Spring Boot properties or such as transform.config. or environment variables like - // TRANSFORM_CONFIG_. - private final Map config = new HashMap<>(); + // Populated from Spring Boot properties or such as transform.config.file. or environment variables like + // TRANSFORM_CONFIG_FILE_. + private final Map files = new HashMap<>(); + + public Map getFile() + { + return files; + } public List retrieveResources() { - return TransformConfigFromFiles.retrieveResources(config); + return TransformConfigFromFiles.retrieveResources(files); } } diff --git a/engines/base/src/main/java/org/alfresco/transform/base/registry/TransformConfigFromFiles.java b/engines/base/src/main/java/org/alfresco/transform/base/registry/TransformConfigFromFiles.java index 518a9f0d..6edf8550 100644 --- a/engines/base/src/main/java/org/alfresco/transform/base/registry/TransformConfigFromFiles.java +++ b/engines/base/src/main/java/org/alfresco/transform/base/registry/TransformConfigFromFiles.java @@ -26,7 +26,7 @@ */ package org.alfresco.transform.base.registry; -import org.alfresco.transform.common.TransformConfigResourceReader; +import org.alfresco.transform.config.reader.TransformConfigResourceReader; import org.alfresco.transform.config.TransformConfig; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.ClassPathResource; diff --git a/engines/base/src/main/java/org/alfresco/transform/base/registry/TransformRegistry.java b/engines/base/src/main/java/org/alfresco/transform/base/registry/TransformRegistry.java index 60f0c249..e6830e35 100644 --- a/engines/base/src/main/java/org/alfresco/transform/base/registry/TransformRegistry.java +++ b/engines/base/src/main/java/org/alfresco/transform/base/registry/TransformRegistry.java @@ -26,12 +26,18 @@ package org.alfresco.transform.base.registry; import org.alfresco.transform.config.TransformConfig; +import org.alfresco.transform.config.TransformOption; +import org.alfresco.transform.config.TransformOptionGroup; +import org.alfresco.transform.config.TransformOptionValue; +import org.alfresco.transform.config.Transformer; import org.alfresco.transform.registry.AbstractTransformRegistry; import org.alfresco.transform.registry.CombinedTransformConfig; +import org.alfresco.transform.registry.Origin; import org.alfresco.transform.registry.TransformCache; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationContext; import org.springframework.context.event.ContextRefreshedEvent; import org.springframework.context.event.EventListener; @@ -44,11 +50,22 @@ import org.springframework.stereotype.Service; import java.util.Comparator; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Supplier; +import java.util.stream.Stream; +import static java.util.Collections.emptyMap; +import static java.util.Objects.isNull; +import static java.util.stream.Collectors.toUnmodifiableMap; +import static java.util.stream.Collectors.toUnmodifiableSet; import static org.alfresco.transform.config.CoreVersionDecorator.setCoreVersionOnSingleStepTransformers; +import static org.springframework.util.CollectionUtils.isEmpty; @Service public class TransformRegistry extends AbstractTransformRegistry @@ -59,30 +76,53 @@ public class TransformRegistry extends AbstractTransformRegistry private String coreVersion; @Autowired private List transformConfigSources; + @Value("${container.isTRouter}") + private boolean isTRouter; private static class Data extends TransformCache { - private TransformConfig transformConfigBeforeIncompleteTransformsAreRemoved; + private TransformConfig transformConfig; + private TransformConfig uncombinedTransformConfig; + private Map> transformerByNameMap; - public TransformConfig getTransformConfigBeforeIncompleteTransformsAreRemoved() + public TransformConfig getTransformConfig() { - return transformConfigBeforeIncompleteTransformsAreRemoved; + return transformConfig; } - public void setTransformConfigBeforeIncompleteTransformsAreRemoved( - TransformConfig transformConfigBeforeIncompleteTransformsAreRemoved) + public void setTransformConfig(TransformConfig transformConfig) { - this.transformConfigBeforeIncompleteTransformsAreRemoved = transformConfigBeforeIncompleteTransformsAreRemoved; + this.transformConfig = transformConfig; + } + + public TransformConfig getUncombinedTransformConfig() + { + return uncombinedTransformConfig; + } + + public void setUncombinedTransformConfig(TransformConfig uncombinedTransformConfig) + { + this.uncombinedTransformConfig = uncombinedTransformConfig; + } + + public Map> getTransformerByNameMap() + { + return transformerByNameMap; + } + + public void setTransformerByNameMap(Map> transformerByNameMap) + { + this.transformerByNameMap = transformerByNameMap; } } private Data data = new Data(); // Ensures that read operations are blocked while config is being updated - private ReadWriteLock configRefreshLock = new ReentrantReadWriteLock(); + private final ReadWriteLock configRefreshLock = new ReentrantReadWriteLock(); @EventListener(ContextRefreshedEvent.class) - void handleContextRefreshedEvent(final ContextRefreshedEvent event) + public void handleContextRefreshedEvent(final ContextRefreshedEvent event) { final ApplicationContext context = event.getApplicationContext(); // the local "initEngineConfigs" method has to be called through the Spring proxy @@ -135,14 +175,27 @@ public class TransformRegistry extends AbstractTransformRegistry this); }); - TransformConfig transformConfigBeforeIncompleteTransformsAreRemoved = combinedTransformConfig.buildTransformConfig(); + TransformConfig uncombinedTransformConfig = combinedTransformConfig.buildTransformConfig(); combinedTransformConfig.combineTransformerConfig(this); - concurrentUpdate(combinedTransformConfig, transformConfigBeforeIncompleteTransformsAreRemoved); + TransformConfig transformConfig = combinedTransformConfig.buildTransformConfig(); + Map> transformerByNameMap = combinedTransformConfig.getTransformerByNameMap(); + concurrentUpdate(combinedTransformConfig, uncombinedTransformConfig, transformConfig, transformerByNameMap); } public TransformConfig getTransformConfig() { - return getData().getTransformConfigBeforeIncompleteTransformsAreRemoved(); + Data data = getData(); + return isTRouter + ? data.getTransformConfig() + : data.getUncombinedTransformConfig(); + } + + /** + * @return Returns true if transform information has been loaded. + */ + public boolean isReadyForTransformRequests() + { + return getData().getTransforms().size() > 0; } @Override @@ -155,13 +208,16 @@ public class TransformRegistry extends AbstractTransformRegistry * Lock for reads while updating, use {@link #concurrentRead} to access locked fields */ private void concurrentUpdate(CombinedTransformConfig combinedTransformConfig, - TransformConfig transformConfigBeforeIncompleteTransformsAreRemoved) + TransformConfig uncombinedTransformConfig, TransformConfig transformConfig, + Map> transformerByNameMap) { configRefreshLock.writeLock().lock(); try { data = new Data(); // clear data - data.setTransformConfigBeforeIncompleteTransformsAreRemoved(transformConfigBeforeIncompleteTransformsAreRemoved); + data.setTransformConfig(transformConfig); + data.setUncombinedTransformConfig(uncombinedTransformConfig); + data.setTransformerByNameMap(transformerByNameMap); combinedTransformConfig.registerCombinedTransformers(this); } finally @@ -195,4 +251,86 @@ public class TransformRegistry extends AbstractTransformRegistry { logger.warn(msg); } + + public Transformer getTransformer(final String sourceMediaType, final Long fileSizeBytes, + final String targetMediaType, final Map transformOptions) + { + return concurrentRead(() -> + { + long fileSize = fileSizeBytes == null ? 0 : fileSizeBytes; + String transformerName = findTransformerName(sourceMediaType, fileSize, targetMediaType, transformOptions, null); + return getTransformer(transformerName); + }); + } + + public Transformer getTransformer(String transformerName) + { + return getTransformer(getData(), transformerName); + } + + private Transformer getTransformer(Data data, String transformerName) + { + return data.getTransformerByNameMap().get(transformerName).get(); + } + + public boolean checkSourceSize(String transformerName, String sourceMediaType, Long sourceSize, String targetMediaType) + { + return Optional.ofNullable(getTransformer(transformerName)). + map(transformer -> transformer.getSupportedSourceAndTargetList().stream(). + filter(supported -> supported.getSourceMediaType().equals(sourceMediaType) && + supported.getTargetMediaType().equals(targetMediaType)). + findFirst(). + map(supported -> supported.getMaxSourceSizeBytes() == -1 || + supported.getMaxSourceSizeBytes() >= sourceSize). + orElse(false)). + orElse(false); + } + + public String getEngineName(String transformerName) + { + return getData().getTransformerByNameMap().get(transformerName).getReadFrom(); + } + + /** + * Filters the transform options for a given transformer. In a pipeline there may be options for different steps. + */ + public Map filterOptions(final String transformerName, final Map options) + { + Data data = getData(); + final Map> configOptions = data.getTransformConfig().getTransformOptions(); + final Transformer transformer = getTransformer(data, transformerName); + if (isNull(transformer) || isEmpty(options) || isEmpty(configOptions)) + { + return emptyMap(); + } + + final Set knownOptions = transformer.getTransformOptions() + .stream() + .flatMap(name -> configOptions.get(name).stream()) + .filter(Objects::nonNull) + .flatMap(TransformRegistry::retrieveOptionsStrings) + .collect(toUnmodifiableSet()); + if (isEmpty(knownOptions)) + { + return emptyMap(); + } + + return options + .entrySet() + .stream() + .filter(e -> knownOptions.contains(e.getKey())) + .collect(toUnmodifiableMap(Entry::getKey, Entry::getValue)); + } + + private static Stream retrieveOptionsStrings(final TransformOption option) + { + if (option instanceof TransformOptionGroup) + { + return ((TransformOptionGroup) option) + .getTransformOptions() + .stream() + .flatMap(TransformRegistry::retrieveOptionsStrings); + } + return Stream.of(((TransformOptionValue) option).getName()); + } } diff --git a/engines/base/src/main/java/org/alfresco/transform/base/transform/ProcessHandler.java b/engines/base/src/main/java/org/alfresco/transform/base/transform/ProcessHandler.java index 27142953..7487b197 100644 --- a/engines/base/src/main/java/org/alfresco/transform/base/transform/ProcessHandler.java +++ b/engines/base/src/main/java/org/alfresco/transform/base/transform/ProcessHandler.java @@ -184,31 +184,17 @@ abstract class ProcessHandler extends FragmentHandler private String getTransformerName(final String sourceMimetype, long sourceSizeInBytes, final String targetMimetype, final Map transformOptions) { - // The transformOptions always contains sourceEncoding when sent to a T-Engine, even though it should not be - // used to select a transformer. Similar to source and target mimetypes and extensions, but these are not - // passed in transformOptions. - String sourceEncoding = transformOptions.remove(SOURCE_ENCODING); - try + final String transformerName = transformRegistry.findTransformerName(sourceMimetype, + sourceSizeInBytes, targetMimetype, transformOptions, null); + if (transformerName == null) { - final String transformerName = transformRegistry.findTransformerName(sourceMimetype, - sourceSizeInBytes, targetMimetype, transformOptions, null); - if (transformerName == null) - { - throw new TransformException(BAD_REQUEST, "No transforms for: "+ - sourceMimetype+" -> "+targetMimetype+transformOptions.entrySet().stream() - .map(entry -> entry.getKey()+"="+entry.getValue()) - .collect(Collectors.joining(", ", " ", ""))); - } - return transformerName; + throw new TransformException(BAD_REQUEST, "No transforms for: "+ + sourceMimetype+" -> "+targetMimetype+transformOptions.entrySet().stream() + .map(entry -> entry.getKey()+"="+entry.getValue()) + .collect(Collectors.joining(", ", " ", ""))); } - finally - { - if (sourceEncoding != null) - { - transformOptions.put(SOURCE_ENCODING, sourceEncoding); - } - } - } + return transformerName; +} private CustomTransformer getCustomTransformer(String transformName) { diff --git a/engines/base/src/main/resources/application.yaml b/engines/base/src/main/resources/application.yaml index c381bcf8..304ac132 100644 --- a/engines/base/src/main/resources/application.yaml +++ b/engines/base/src/main/resources/application.yaml @@ -22,6 +22,15 @@ server: error: include-message: ALWAYS +# Historic values (AVOID) - Exist to help with transition to newer versions +transformer-routes-path: ${TRANSFORMER_ROUTES_FILE_LOCATION:transformer-pipelines.json} + +logging: + level: + org.alfresco.transform.common.TransformerDebug: debug + +fileStoreUrl: ${FILE_STORE_URL:http://localhost:8099/alfresco/api/-default-/private/sfs/versions/1/file} + transform: core: version: @project.version@ @@ -32,17 +41,6 @@ transform: attempts: 10 timeout: 10 # seconds -# Historic values (AVOID) - Exist to help with transition to newer versions -transformer-routes-path: ${TRANSFORMER_ROUTES_FILE_LOCATION:transformer-pipelines.json} - -logging: - level: - # org.alfresco.util.exec.RuntimeExec: debug - # org.alfresco.transform.base.metadataExtractors: debug - org.alfresco.transform.common.TransformerDebug: debug - -fileStoreUrl: ${FILE_STORE_URL:http://localhost:8099/alfresco/api/-default-/private/sfs/versions/1/file} - jms-listener: concurrency: ${JMS_LISTENER_CONCURRENCY:1-10} @@ -62,12 +60,5 @@ management: container: name: ${HOSTNAME:t-engine} - behind-ingres: false - -async-task-executor: - core-pool-size: 1 - max-pool-size: 2147483647 - keep-alive-seconds: 60 - queue-capacity: 2147483647 - allow-core-thread-time-out: false - prestart-all-core-threads: false \ No newline at end of file + isTRouter: false + behind-ingres: false \ No newline at end of file diff --git a/engines/base/src/test/java/org/alfresco/transform/base/TransformControllerAllInOneTest.java b/engines/base/src/test/java/org/alfresco/transform/base/TransformControllerAllInOneTest.java index d4c165ac..ccd86257 100644 --- a/engines/base/src/test/java/org/alfresco/transform/base/TransformControllerAllInOneTest.java +++ b/engines/base/src/test/java/org/alfresco/transform/base/TransformControllerAllInOneTest.java @@ -124,7 +124,7 @@ public class TransformControllerAllInOneTest { mockMvc.perform(MockMvcRequestBuilders.get(ENDPOINT_VERSION)) .andExpect(status().isOk()) - .andExpect(content().string("AllInOne "+coreVersion+" available")); + .andExpect(content().string("AllInOne "+coreVersion)); } @Test diff --git a/engines/base/src/test/java/org/alfresco/transform/base/TransformControllerTest.java b/engines/base/src/test/java/org/alfresco/transform/base/TransformControllerTest.java index e2b8e6e7..7716dd27 100644 --- a/engines/base/src/test/java/org/alfresco/transform/base/TransformControllerTest.java +++ b/engines/base/src/test/java/org/alfresco/transform/base/TransformControllerTest.java @@ -206,7 +206,7 @@ public class TransformControllerTest { mockMvc.perform(MockMvcRequestBuilders.get(ENDPOINT_VERSION)) .andExpect(status().isOk()) - .andExpect(content().string("TwoCustomTransformers "+coreVersion+" available")); + .andExpect(content().string("TwoCustomTransformers "+coreVersion)); } @Test diff --git a/engines/base/src/test/java/org/alfresco/transform/base/registry/TransformRegistryRefreshTest.java b/engines/base/src/test/java/org/alfresco/transform/base/registry/TransformRegistryRefreshTest.java index 95011bc0..3e6418b0 100644 --- a/engines/base/src/test/java/org/alfresco/transform/base/registry/TransformRegistryRefreshTest.java +++ b/engines/base/src/test/java/org/alfresco/transform/base/registry/TransformRegistryRefreshTest.java @@ -47,7 +47,7 @@ public class TransformRegistryRefreshTest verify(transformRegistry, atLeast(1)).retrieveConfig(); // As we can't change the content of a classpath resource, lets change what is read. - ReflectionTestUtils.setField(transformConfigFiles, "config", ImmutableMap.of( + ReflectionTestUtils.setField(transformConfigFiles, "files", ImmutableMap.of( "a", "config/addA2B.json", "foo", "config/addB2C.json")); transformConfigFromFiles.initFileConfig(); diff --git a/engines/base/src/test/java/org/alfresco/transform/base/registry/TransformRegistryTest.java b/engines/base/src/test/java/org/alfresco/transform/base/registry/TransformRegistryTest.java index 3bdc0105..208dc0da 100644 --- a/engines/base/src/test/java/org/alfresco/transform/base/registry/TransformRegistryTest.java +++ b/engines/base/src/test/java/org/alfresco/transform/base/registry/TransformRegistryTest.java @@ -45,6 +45,8 @@ import java.util.List; import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; @AutoConfigureMockMvc @SpringBootTest(classes={org.alfresco.transform.base.Application.class}) @@ -68,8 +70,9 @@ public class TransformRegistryTest { transformConfigSources.clear(); ReflectionTestUtils.setField(transformConfigFromTransformEngines, "transformEngines", Collections.emptyList()); - ReflectionTestUtils.setField(transformConfigFiles, "config", Collections.emptyMap()); + ReflectionTestUtils.setField(transformConfigFiles, "files", Collections.emptyMap()); ReflectionTestUtils.setField(transformConfigFilesHistoric, "additional", Collections.emptyMap()); + ReflectionTestUtils.setField(transformRegistry, "isTRouter", false); transformRegistry.retrieveConfig(); } @@ -112,12 +115,46 @@ public class TransformRegistryTest getTransformerNames(transformRegistry.getTransformConfig())); } + @Test + public void uncombinedConfigFromEngine() + { + ReflectionTestUtils.setField(transformConfigFromTransformEngines, "transformEngines", ImmutableList.of( + new FakeTransformEngineWithAllInOne(), + new FakeTransformEngineWithTwoCustomTransformers())); + transformConfigFromTransformEngines.initTransformEngineConfig(); + transformRegistry.retrieveConfig(); + + assertEquals("Pdf2Png, TxT2Pdf, Txt2JpgViaPdf, Txt2PngViaPdf", + getTransformerNames(transformRegistry.getTransformConfig())); + + ReflectionTestUtils.setField(transformRegistry, "isTRouter", true); + transformConfigFromTransformEngines.initTransformEngineConfig(); + transformRegistry.retrieveConfig(); + + assertEquals("Pdf2Png, TxT2Pdf, Txt2PngViaPdf", + getTransformerNames(transformRegistry.getTransformConfig())); + } + + @Test + public void combinedConfigFromRouter() + { + ReflectionTestUtils.setField(transformRegistry, "isTRouter", true); + ReflectionTestUtils.setField(transformConfigFromTransformEngines, "transformEngines", ImmutableList.of( + new FakeTransformEngineWithAllInOne(), + new FakeTransformEngineWithTwoCustomTransformers())); + transformConfigFromTransformEngines.initTransformEngineConfig(); + transformRegistry.retrieveConfig(); + + assertEquals("Pdf2Png, TxT2Pdf, Txt2PngViaPdf", + getTransformerNames(transformRegistry.getTransformConfig())); + } + @Test public void singleTransformEngineWithAdditionalConfig() { ReflectionTestUtils.setField(transformConfigFromTransformEngines, "transformEngines", ImmutableList.of( new FakeTransformEngineWithOneCustomTransformer())); - ReflectionTestUtils.setField(transformConfigFiles, "config", ImmutableMap.of( + ReflectionTestUtils.setField(transformConfigFiles, "files", ImmutableMap.of( "a", "config/addA2B.json", "foo", "config/addB2C.json")); @@ -175,4 +212,19 @@ public class TransformRegistryTest assertEquals("A2Z", getTransformerNames(transformRegistry.getTransformConfig())); } + + @Test + public void isReadyForTransformRequests() + { + transformConfigFromTransformEngines.initTransformEngineConfig(); + transformRegistry.retrieveConfig(); + assertFalse(transformRegistry.isReadyForTransformRequests()); + + ReflectionTestUtils.setField(transformConfigFromTransformEngines, "transformEngines", ImmutableList.of( + new FakeTransformEngineWithOneCustomTransformer())); + transformConfigFromTransformEngines.initTransformEngineConfig(); + transformRegistry.retrieveConfig(); + + assertTrue(transformRegistry.isReadyForTransformRequests()); + } } diff --git a/engines/example/src/main/java/org/alfresco/transform/example/HelloTransformEngine.java b/engines/example/src/main/java/org/alfresco/transform/example/HelloTransformEngine.java index 9122cc02..aa9c36cc 100644 --- a/engines/example/src/main/java/org/alfresco/transform/example/HelloTransformEngine.java +++ b/engines/example/src/main/java/org/alfresco/transform/example/HelloTransformEngine.java @@ -29,7 +29,7 @@ package org.alfresco.transform.example; import com.google.common.collect.ImmutableMap; import org.alfresco.transform.base.TransformEngine; import org.alfresco.transform.base.probes.ProbeTransform; -import org.alfresco.transform.common.TransformConfigResourceReader; +import org.alfresco.transform.config.reader.TransformConfigResourceReader; import org.alfresco.transform.config.TransformConfig; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -43,7 +43,7 @@ public class HelloTransformEngine implements TransformEngine @Override public String getTransformEngineName() { - return "0200_hello"; + return "0200 Hello"; } @Override diff --git a/engines/imagemagick/src/main/java/org/alfresco/transform/imagemagick/ImageMagickTransformEngine.java b/engines/imagemagick/src/main/java/org/alfresco/transform/imagemagick/ImageMagickTransformEngine.java index 3517e448..3b4ed436 100644 --- a/engines/imagemagick/src/main/java/org/alfresco/transform/imagemagick/ImageMagickTransformEngine.java +++ b/engines/imagemagick/src/main/java/org/alfresco/transform/imagemagick/ImageMagickTransformEngine.java @@ -28,7 +28,7 @@ package org.alfresco.transform.imagemagick; import org.alfresco.transform.base.TransformEngine; import org.alfresco.transform.base.probes.ProbeTransform; -import org.alfresco.transform.common.TransformConfigResourceReader; +import org.alfresco.transform.config.reader.TransformConfigResourceReader; import org.alfresco.transform.config.TransformConfig; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -48,7 +48,7 @@ public class ImageMagickTransformEngine implements TransformEngine @Override public String getTransformEngineName() { - return "0030-ImageMagick"; + return "0030 ImageMagick"; } @Override diff --git a/engines/libreoffice/src/main/java/org/alfresco/transform/libreoffice/LibreOfficeTransformEngine.java b/engines/libreoffice/src/main/java/org/alfresco/transform/libreoffice/LibreOfficeTransformEngine.java index 7f94259d..3eb04a21 100644 --- a/engines/libreoffice/src/main/java/org/alfresco/transform/libreoffice/LibreOfficeTransformEngine.java +++ b/engines/libreoffice/src/main/java/org/alfresco/transform/libreoffice/LibreOfficeTransformEngine.java @@ -28,7 +28,7 @@ package org.alfresco.transform.libreoffice; import org.alfresco.transform.base.TransformEngine; import org.alfresco.transform.base.probes.ProbeTransform; -import org.alfresco.transform.common.TransformConfigResourceReader; +import org.alfresco.transform.config.reader.TransformConfigResourceReader; import org.alfresco.transform.config.TransformConfig; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -48,7 +48,7 @@ public class LibreOfficeTransformEngine implements TransformEngine @Override public String getTransformEngineName() { - return "0020-LibreOffice"; + return "0020 LibreOffice"; } @Override diff --git a/engines/misc/pom.xml b/engines/misc/pom.xml index e4d473a5..6c627ac6 100644 --- a/engines/misc/pom.xml +++ b/engines/misc/pom.xml @@ -12,8 +12,8 @@ - alfresco/alfresco-transform-misc quay.io + alfresco/alfresco-transform-misc ${project.artifactId} diff --git a/engines/misc/src/main/java/org/alfresco/transform/misc/MiscTransformEngine.java b/engines/misc/src/main/java/org/alfresco/transform/misc/MiscTransformEngine.java index 1f66c3c1..6fc0276a 100644 --- a/engines/misc/src/main/java/org/alfresco/transform/misc/MiscTransformEngine.java +++ b/engines/misc/src/main/java/org/alfresco/transform/misc/MiscTransformEngine.java @@ -29,7 +29,7 @@ package org.alfresco.transform.misc; import com.google.common.collect.ImmutableMap; import org.alfresco.transform.base.TransformEngine; import org.alfresco.transform.base.probes.ProbeTransform; -import org.alfresco.transform.common.TransformConfigResourceReader; +import org.alfresco.transform.config.reader.TransformConfigResourceReader; import org.alfresco.transform.config.TransformConfig; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -52,7 +52,7 @@ public class MiscTransformEngine implements TransformEngine @Override public String getTransformEngineName() { - return "0050-Misc"; + return "0050 Misc"; } @Override diff --git a/engines/pdfrenderer/src/main/java/org/alfresco/transform/pdfrenderer/PdfRendererTransformEngine.java b/engines/pdfrenderer/src/main/java/org/alfresco/transform/pdfrenderer/PdfRendererTransformEngine.java index 12a05ee3..c848ed80 100644 --- a/engines/pdfrenderer/src/main/java/org/alfresco/transform/pdfrenderer/PdfRendererTransformEngine.java +++ b/engines/pdfrenderer/src/main/java/org/alfresco/transform/pdfrenderer/PdfRendererTransformEngine.java @@ -28,7 +28,7 @@ package org.alfresco.transform.pdfrenderer; import org.alfresco.transform.base.TransformEngine; import org.alfresco.transform.base.probes.ProbeTransform; -import org.alfresco.transform.common.TransformConfigResourceReader; +import org.alfresco.transform.config.reader.TransformConfigResourceReader; import org.alfresco.transform.config.TransformConfig; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -48,7 +48,7 @@ public class PdfRendererTransformEngine implements TransformEngine @Override public String getTransformEngineName() { - return "0040-PdfRenderer"; + return "0040 PdfRenderer"; } @Override diff --git a/engines/tika/src/main/java/org/alfresco/transform/tika/TikaTransformEngine.java b/engines/tika/src/main/java/org/alfresco/transform/tika/TikaTransformEngine.java index b548d583..123f42b1 100644 --- a/engines/tika/src/main/java/org/alfresco/transform/tika/TikaTransformEngine.java +++ b/engines/tika/src/main/java/org/alfresco/transform/tika/TikaTransformEngine.java @@ -28,7 +28,7 @@ package org.alfresco.transform.tika; import org.alfresco.transform.base.TransformEngine; import org.alfresco.transform.base.probes.ProbeTransform; -import org.alfresco.transform.common.TransformConfigResourceReader; +import org.alfresco.transform.config.reader.TransformConfigResourceReader; import org.alfresco.transform.config.TransformConfig; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -48,7 +48,7 @@ public class TikaTransformEngine implements TransformEngine @Override public String getTransformEngineName() { - return "0010-Tika"; + return "0010 Tika"; } @Override diff --git a/model/pom.xml b/model/pom.xml index fe1e3ac7..13edb97f 100644 --- a/model/pom.xml +++ b/model/pom.xml @@ -37,6 +37,10 @@ jackson-annotations provided + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + junit junit diff --git a/model/src/main/java/org/alfresco/transform/common/TransformerDebug.java b/model/src/main/java/org/alfresco/transform/common/TransformerDebug.java index a88324a4..6207927b 100644 --- a/model/src/main/java/org/alfresco/transform/common/TransformerDebug.java +++ b/model/src/main/java/org/alfresco/transform/common/TransformerDebug.java @@ -50,7 +50,7 @@ public class TransformerDebug private static int MAX_OPTION_END_CHARS = 5; private static String MAX_OPTION_DOTS = "..."; - private boolean isTEngine = false; + private boolean isTRouter = false; public void pushTransform(TransformRequest request) { @@ -64,7 +64,7 @@ public class TransformerDebug String message = getPaddedReference(reference) + getMimetypeExt(step.getSourceMediaType()) + getTargetMimetypeExt(step.getTargetMediaType(), step.getSourceMediaType()) + ' ' + - (isTopLevel || isTEngine() + (isTopLevel || !isTRouter() ? fileSize(request.getSourceSize()) + ' ' + getRenditionName(new RepositoryClientData(request.getClientData()).getRenditionName()) : "") + @@ -117,7 +117,7 @@ public class TransformerDebug if (logger.isDebugEnabled()) { String message = getPaddedReference(reference) + "Finished in " + ms(elapsedTime); - if (isTopLevel(reference) || isTEngine()) + if (isTopLevel(reference) || !isTRouter()) { logger.debug(message); } @@ -221,9 +221,9 @@ public class TransformerDebug } // T-Engines call this method, as the T-Router will appended the same debug messages - public TransformerDebug setIsTEngine(boolean isTEngine) + public TransformerDebug setIsTRouter(boolean isTRouter) { - this.isTEngine = isTEngine; + this.isTRouter = isTRouter; return this; } @@ -234,12 +234,12 @@ public class TransformerDebug private boolean isDebugToBeReturned(RepositoryClientData repositoryClientData) { - return !isTEngine() && repositoryClientData.isDebugRequested(); + return isTRouter() && repositoryClientData.isDebugRequested(); } - private boolean isTEngine() + private boolean isTRouter() { - return isTEngine; + return isTRouter; } private boolean isTopLevel(String reference) diff --git a/model/src/main/java/org/alfresco/transform/config/TransformConfig.java b/model/src/main/java/org/alfresco/transform/config/TransformConfig.java index 89fece65..d1b31e8e 100644 --- a/model/src/main/java/org/alfresco/transform/config/TransformConfig.java +++ b/model/src/main/java/org/alfresco/transform/config/TransformConfig.java @@ -56,7 +56,32 @@ public class TransformConfig public void setTransformOptions(Map> transformOptions) { - this.transformOptions = new HashMap<>(transformOptions); + this.transformOptions = transformOptions == null ? new HashMap<>() : new HashMap<>(transformOptions); + } + + public void setRemoveTransformers(Set removeTransformers) + { + this.removeTransformers = removeTransformers == null ? new HashSet<>() : removeTransformers; + } + + public void setAddSupported(Set addSupported) + { + this.addSupported = addSupported == null ? new HashSet<>() : addSupported; + } + + public void setRemoveSupported(Set removeSupported) + { + this.removeSupported = removeSupported == null ? new HashSet<>() : removeSupported; + } + + public void setOverrideSupported(Set overrideSupported) + { + this.overrideSupported = overrideSupported == null ? new HashSet<>() : overrideSupported; + } + + public void setSupportedDefaults(Set supportedDefaults) + { + this.supportedDefaults = supportedDefaults == null ? new HashSet<>() : supportedDefaults; } public List getTransformers() @@ -91,7 +116,7 @@ public class TransformConfig public void setTransformers(List transformers) { - this.transformers = transformers; + this.transformers = transformers == null ? new ArrayList<>() : transformers; } @Override diff --git a/model/src/main/java/org/alfresco/transform/config/TransformOptionGroup.java b/model/src/main/java/org/alfresco/transform/config/TransformOptionGroup.java index 54225ff8..456dc26f 100644 --- a/model/src/main/java/org/alfresco/transform/config/TransformOptionGroup.java +++ b/model/src/main/java/org/alfresco/transform/config/TransformOptionGroup.java @@ -53,7 +53,7 @@ public class TransformOptionGroup extends AbstractTransformOption public void setTransformOptions(Set transformOptions) { - this.transformOptions = transformOptions; + this.transformOptions = transformOptions == null ? new HashSet<>() : transformOptions; } @Override diff --git a/model/src/main/java/org/alfresco/transform/config/Transformer.java b/model/src/main/java/org/alfresco/transform/config/Transformer.java index 6777f6c6..6404264c 100644 --- a/model/src/main/java/org/alfresco/transform/config/Transformer.java +++ b/model/src/main/java/org/alfresco/transform/config/Transformer.java @@ -119,7 +119,7 @@ public class Transformer public void setTransformerPipeline(List transformerPipeline) { - this.transformerPipeline = transformerPipeline; + this.transformerPipeline = transformerPipeline == null ? new ArrayList<>() : transformerPipeline; } public List getTransformerFailover() @@ -129,7 +129,7 @@ public class Transformer public void setTransformerFailover(List transformerFailover) { - this.transformerFailover = transformerFailover; + this.transformerFailover = transformerFailover == null ? new ArrayList<>() : transformerFailover; } public Set getTransformOptions() @@ -139,7 +139,7 @@ public class Transformer public void setTransformOptions(Set transformOptions) { - this.transformOptions = transformOptions; + this.transformOptions = transformOptions == null ? new HashSet<>() : transformOptions; } public Set getSupportedSourceAndTargetList() @@ -150,7 +150,7 @@ public class Transformer public void setSupportedSourceAndTargetList( Set supportedSourceAndTargetList) { - this.supportedSourceAndTargetList = supportedSourceAndTargetList; + this.supportedSourceAndTargetList = supportedSourceAndTargetList == null ? new HashSet<>() : supportedSourceAndTargetList; } @Override diff --git a/model/src/main/java/org/alfresco/transform/config/reader/TransformConfigReader.java b/model/src/main/java/org/alfresco/transform/config/reader/TransformConfigReader.java new file mode 100644 index 00000000..ee52385a --- /dev/null +++ b/model/src/main/java/org/alfresco/transform/config/reader/TransformConfigReader.java @@ -0,0 +1,31 @@ +/* + * #%L + * Alfresco Transform Model + * %% + * Copyright (C) 2015 - 2022 Alfresco Software Limited + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * . + * #L% + */ +package org.alfresco.transform.config.reader; + +import org.alfresco.transform.config.TransformConfig; + +import java.io.IOException; + +public interface TransformConfigReader +{ + TransformConfig load() throws IOException; +} diff --git a/model/src/main/java/org/alfresco/transform/config/reader/TransformConfigReaderFactory.java b/model/src/main/java/org/alfresco/transform/config/reader/TransformConfigReaderFactory.java new file mode 100644 index 00000000..17c8cc27 --- /dev/null +++ b/model/src/main/java/org/alfresco/transform/config/reader/TransformConfigReaderFactory.java @@ -0,0 +1,53 @@ +/* + * #%L + * Alfresco Transform Model + * %% + * Copyright (C) 2015 - 2022 Alfresco Software Limited + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * . + * #L% + */ +package org.alfresco.transform.config.reader; + +import static java.util.Objects.isNull; + +import org.springframework.core.io.Resource; + +public class TransformConfigReaderFactory +{ + public static TransformConfigReader create(final Resource resource) + { + final String fileName = resource.getFilename(); + + if (isNull(fileName) || !fileName.contains(".")) + { + throw new RuntimeException("Invalid configuration file: " + fileName); + } + final String extension = fileName.substring(fileName.lastIndexOf('.') + 1); + switch (extension) + { + case "properties": + throw new UnsupportedOperationException(".properties configuration files are no longer " + + "supported: " + fileName); + case "yaml": + case "yml": + return new TransformConfigReaderYaml(resource); + case "json": + return new TransformConfigReaderJson(resource); + default: + throw new RuntimeException("Unknown configuration file type: " + fileName); + } + } +} diff --git a/model/src/main/java/org/alfresco/transform/config/reader/TransformConfigReaderJson.java b/model/src/main/java/org/alfresco/transform/config/reader/TransformConfigReaderJson.java new file mode 100644 index 00000000..ad7daed0 --- /dev/null +++ b/model/src/main/java/org/alfresco/transform/config/reader/TransformConfigReaderJson.java @@ -0,0 +1,47 @@ +/* + * #%L + * Alfresco Transform Model + * %% + * Copyright (C) 2015 - 2022 Alfresco Software Limited + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * . + * #L% + */ +package org.alfresco.transform.config.reader; + +import java.io.IOException; + +import org.alfresco.transform.config.TransformConfig; +import org.springframework.core.io.Resource; + +import com.fasterxml.jackson.databind.ObjectMapper; + +public class TransformConfigReaderJson implements TransformConfigReader +{ + private static final ObjectMapper MAPPER= new ObjectMapper(); + + private final Resource resource; + + TransformConfigReaderJson(final Resource resource) + { + this.resource = resource; + } + + @Override + public TransformConfig load() throws IOException + { + return MAPPER.readValue(resource.getInputStream(), TransformConfig.class); + } +} \ No newline at end of file diff --git a/model/src/main/java/org/alfresco/transform/config/reader/TransformConfigReaderYaml.java b/model/src/main/java/org/alfresco/transform/config/reader/TransformConfigReaderYaml.java new file mode 100644 index 00000000..093e91c5 --- /dev/null +++ b/model/src/main/java/org/alfresco/transform/config/reader/TransformConfigReaderYaml.java @@ -0,0 +1,48 @@ +/* + * #%L + * Alfresco Transform Model + * %% + * Copyright (C) 2015 - 2022 Alfresco Software Limited + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * . + * #L% + */ +package org.alfresco.transform.config.reader; + +import java.io.IOException; + +import org.alfresco.transform.config.TransformConfig; +import org.springframework.core.io.Resource; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; + +public class TransformConfigReaderYaml implements TransformConfigReader +{ + private static final ObjectMapper MAPPER = new ObjectMapper(new YAMLFactory()); + + private final Resource resource; + + TransformConfigReaderYaml(final Resource resource) + { + this.resource = resource; + } + + @Override + public TransformConfig load() throws IOException + { + return MAPPER.readValue(resource.getInputStream(), TransformConfig.class); + } +} diff --git a/model/src/main/java/org/alfresco/transform/common/TransformConfigResourceReader.java b/model/src/main/java/org/alfresco/transform/config/reader/TransformConfigResourceReader.java similarity index 72% rename from model/src/main/java/org/alfresco/transform/common/TransformConfigResourceReader.java rename to model/src/main/java/org/alfresco/transform/config/reader/TransformConfigResourceReader.java index 8c95b167..02fd5c06 100644 --- a/model/src/main/java/org/alfresco/transform/common/TransformConfigResourceReader.java +++ b/model/src/main/java/org/alfresco/transform/config/reader/TransformConfigResourceReader.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2022 Alfresco Software Limited + * Copyright (C) 2022 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,9 +24,9 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transform.common; +package org.alfresco.transform.config.reader; -import com.fasterxml.jackson.databind.ObjectMapper; +import org.alfresco.transform.common.TransformException; import org.alfresco.transform.config.TransformConfig; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.Resource; @@ -41,7 +41,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR; /** - * Reads {@link TransformConfig} from a {@json} file. Typically used by {@code TransformEngine.getTransformConfig()}. + * Reads {@link TransformConfig} from json or yaml files. Typically used by {@code TransformEngine.getTransformConfig()}. * 
  *     transformConfigResourceReader.read("classpath:pdfrenderer_engine_config.json");
  *
@@ -51,20 +51,16 @@ public class TransformConfigResourceReader { @Autowired ResourceLoader resourceLoader; - private ObjectMapper jsonObjectMapper = new ObjectMapper(); - - public TransformConfig read(String engineConfigLocation) + public TransformConfig read(String resourcePath) { - Resource engineConfig = resourceLoader.getResource(engineConfigLocation); - return read(engineConfig); + return read(resourceLoader.getResource(resourcePath)); } public TransformConfig read(Resource resource) { - try (Reader reader = new InputStreamReader(resource.getInputStream(), UTF_8)) + try { - TransformConfig transformConfig = jsonObjectMapper.readValue(reader, TransformConfig.class); - return transformConfig; + return TransformConfigReaderFactory.create(resource).load(); } catch (IOException e) { diff --git a/model/src/main/java/org/alfresco/transform/registry/CombinedTransformConfig.java b/model/src/main/java/org/alfresco/transform/registry/CombinedTransformConfig.java index f11421de..51f93071 100644 --- a/model/src/main/java/org/alfresco/transform/registry/CombinedTransformConfig.java +++ b/model/src/main/java/org/alfresco/transform/registry/CombinedTransformConfig.java @@ -32,6 +32,7 @@ import org.alfresco.transform.config.TransformStep; import org.alfresco.transform.config.Transformer; import org.alfresco.transform.config.TransformerAndTypes; import org.alfresco.transform.config.Types; +import org.apache.commons.lang3.tuple.Triple; import java.util.ArrayList; import java.util.HashMap; @@ -827,4 +828,9 @@ public class CombinedTransformConfig { return combinedTransformers.size(); } + + public Map> getTransformerByNameMap() + { + return combinedTransformers.stream().collect(Collectors.toMap(origin -> origin.get().getTransformerName(), origin -> origin)); + } } diff --git a/model/src/main/java/org/alfresco/transform/registry/TransformRegistryHelper.java b/model/src/main/java/org/alfresco/transform/registry/TransformRegistryHelper.java index 3cff2c71..e5ee4e0d 100644 --- a/model/src/main/java/org/alfresco/transform/registry/TransformRegistryHelper.java +++ b/model/src/main/java/org/alfresco/transform/registry/TransformRegistryHelper.java @@ -39,6 +39,7 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; import static java.util.Map.Entry; import static java.util.stream.Collectors.toMap; +import static org.alfresco.transform.common.RequestParamMap.SOURCE_ENCODING; import static org.springframework.http.HttpStatus.BAD_REQUEST; class TransformRegistryHelper @@ -95,17 +96,36 @@ class TransformRegistryHelper return cachedTransformList; } - final List builtTransformList = buildTransformList(data, - sourceMimetype, - targetMimetype, - filterTimeout(actualOptions)); - - if (renditionName != null) + // The transformOptions always contains sourceEncoding and possibly timeout when sent to a T-Engine, even though + // they should not be used to select a transformer. Would like to change this, but cannot as we need to support + // older ACS repo versions. + String sourceEncoding = actualOptions.remove(SOURCE_ENCODING); + String timeout = actualOptions.remove(TIMEOUT); + try { - data.cache(renditionName, sourceMimetype, builtTransformList); - } + final List builtTransformList = buildTransformList(data, + sourceMimetype, + targetMimetype, + actualOptions); - return builtTransformList; + if (renditionName != null) + { + data.cache(renditionName, sourceMimetype, builtTransformList); + } + + return builtTransformList; + } + finally + { + if (sourceEncoding != null) + { + actualOptions.put(SOURCE_ENCODING, sourceEncoding); + } + if (timeout != null) + { + actualOptions.put(TIMEOUT, timeout); + } + } } private static List buildTransformList( @@ -349,18 +369,4 @@ class TransformRegistryHelper .stream() .allMatch(transformOptions::containsKey); } - - private static Map filterTimeout(final Map options) - { - // Remove the "timeout" property from the actualOptions as it is not used to select a transformer. - if (!options.containsKey(TIMEOUT)) - { - return options; - } - return options - .entrySet() - .stream() - .filter(e -> !TIMEOUT.equals(e.getKey())) - .collect(toMap(Entry::getKey, Entry::getValue)); - } } diff --git a/model/src/test/java/org/alfresco/transform/common/TransformerDebugTest.java b/model/src/test/java/org/alfresco/transform/common/TransformerDebugTest.java index ee7e9480..48f15056 100644 --- a/model/src/test/java/org/alfresco/transform/common/TransformerDebugTest.java +++ b/model/src/test/java/org/alfresco/transform/common/TransformerDebugTest.java @@ -63,10 +63,10 @@ class TransformerDebugTest .replaceAll(" [\\d,]+ ms", " -- ms"); } - private void twoStepTransform(boolean isTEngine, boolean fail, Level logLevel, String renditionName, + private void twoStepTransform(boolean isTRouter, boolean fail, Level logLevel, String renditionName, long sourceSize) { - transformerDebug.setIsTEngine(isTEngine); + transformerDebug.setIsTRouter(isTRouter); monitorLogs(logLevel); TransformRequest request = TransformRequest.builder() @@ -144,7 +144,7 @@ class TransformerDebugTest @Test void testRouterTwoStepTransform() { - twoStepTransform(false, false, Level.DEBUG, "", 1234L); + twoStepTransform(true, false, Level.DEBUG, "", 1234L); Assertions.assertEquals("" + "1 txt pdf 1.2 KB wrapper\n" + @@ -159,7 +159,7 @@ class TransformerDebugTest @Test void testRouterTwoStepTransformWithTrace() { - twoStepTransform(false, false, Level.TRACE, "", 1234L); + twoStepTransform(true, false, Level.TRACE, "", 1234L); // With trace there are "Finished" lines for nested transforms, like a T-Engine's debug but still without // the size and rendition name @@ -178,7 +178,7 @@ class TransformerDebugTest @Test void testEngineTwoStepTransform() { - twoStepTransform(true, false, Level.DEBUG, "", 1234L); + twoStepTransform(false, false, Level.DEBUG, "", 1234L); // Note the first and last lines would only ever be logged on the router, but the expected data includes // the extra "Finished" lines, sizes and renditions (if set in client data). @@ -197,7 +197,7 @@ class TransformerDebugTest @Test void testRouterTwoStepTransformWithFailure() { - twoStepTransform(false, true, Level.DEBUG, "", 1234L); + twoStepTransform(true, true, Level.DEBUG, "", 1234L); Assertions.assertEquals("" + "1 txt pdf 1.2 KB wrapper\n" + @@ -212,7 +212,7 @@ class TransformerDebugTest @Test void testRenditionName() { - twoStepTransform(false, false, Level.DEBUG, "renditionName", 1234L); + twoStepTransform(true, false, Level.DEBUG, "renditionName", 1234L); Assertions.assertEquals("" + "1 txt pdf 1.2 KB -- renditionName -- wrapper\n" + @@ -227,7 +227,7 @@ class TransformerDebugTest @Test void testMetadataExtract() { - twoStepTransform(false, false, Level.DEBUG, "transform:alfresco-metadata-extract", 1234L); + twoStepTransform(true, false, Level.DEBUG, "transform:alfresco-metadata-extract", 1234L); Assertions.assertEquals("" + "1 txt pdf 1.2 KB -- metadataExtract -- wrapper\n" + @@ -242,7 +242,7 @@ class TransformerDebugTest @Test void testMetadataEmbed() { - twoStepTransform(false, false, Level.DEBUG, "transform:alfresco-metadata-embed", 1234L); + twoStepTransform(true, false, Level.DEBUG, "transform:alfresco-metadata-embed", 1234L); Assertions.assertEquals("" + "1 txt pdf 1.2 KB -- metadataEmbed -- wrapper\n" + @@ -257,7 +257,7 @@ class TransformerDebugTest @Test void testSourceSize1Byte() { - twoStepTransform(false, false, Level.DEBUG, "", 1); + twoStepTransform(true, false, Level.DEBUG, "", 1); Assertions.assertEquals("" + "1 txt pdf 1 byte wrapper\n" + @@ -272,7 +272,7 @@ class TransformerDebugTest @Test void testSourceSize23TB() { - twoStepTransform(false, false, Level.DEBUG, "", 23L*1024*1024*1024*1024); + twoStepTransform(true, false, Level.DEBUG, "", 23L*1024*1024*1024*1024); Assertions.assertEquals("" + "1 txt pdf 23 TB wrapper\n" + @@ -285,7 +285,7 @@ class TransformerDebugTest } @Test - void testLogFailure() + void testLogFailureOnTEngine() { monitorLogs(Level.TRACE); @@ -296,11 +296,33 @@ class TransformerDebugTest .withClientData(origClientData) .build(); + transformerDebug.setIsTRouter(false); transformerDebug.logFailure(reply); String expectedDebug = " T-Request was null - a major error"; Assertions.assertEquals(expectedDebug, getTransformerDebugOutput()); - assertEquals(origClientData+DEBUG_SEPARATOR+expectedDebug, reply.getClientData()); + assertEquals(origClientData, reply.getClientData()); + } + + @Test + void testLogFailureOnTRouter() + { + monitorLogs(Level.TRACE); + + String origClientData = clientDataWithDebugRequest(""); + TransformReply reply = TransformReply.builder() + .withInternalContext(InternalContext.initialise(null)) + .withErrorDetails("T-Request was null - a major error") + .withClientData(origClientData) + .build(); + + transformerDebug.setIsTRouter(true); + transformerDebug.logFailure(reply); + + String expectedDebug = " T-Request was null - a major error"; + String expectedClientData = origClientData+DEBUG_SEPARATOR+expectedDebug; + Assertions.assertEquals(expectedDebug, getTransformerDebugOutput()); + assertEquals(expectedClientData, reply.getClientData()); } @Test diff --git a/model/src/test/java/org/alfresco/transform/config/reader/TransformConfigReaderJsonTest.java b/model/src/test/java/org/alfresco/transform/config/reader/TransformConfigReaderJsonTest.java new file mode 100644 index 00000000..1c824946 --- /dev/null +++ b/model/src/test/java/org/alfresco/transform/config/reader/TransformConfigReaderJsonTest.java @@ -0,0 +1,300 @@ +/* + * #%L + * Alfresco Transform Model + * %% + * Copyright (C) 2015 - 2022 Alfresco Software Limited + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * . + * #L% + */ +package org.alfresco.transform.config.reader; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import com.google.common.collect.ImmutableMap; +import org.alfresco.transform.config.TransformConfig; +import org.alfresco.transform.config.TransformOption; +import org.alfresco.transform.config.TransformOptionGroup; +import org.alfresco.transform.config.TransformOptionValue; +import org.alfresco.transform.config.Transformer; +import org.alfresco.transform.config.SupportedSourceAndTarget; +import org.alfresco.transform.config.TransformStep; +import org.junit.jupiter.api.Test; +import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.Resource; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +public class TransformConfigReaderJsonTest +{ + @Test + public void testEmptyRoutesFile() throws Exception + { + final Resource resource = new ClassPathResource("config/sample1.json"); + final TransformConfigReader loader = TransformConfigReaderFactory.create(resource); + TransformConfig transformConfig = loader.load(); + final List transformers = transformConfig.getTransformers(); + + assertNotNull(transformers); + assertEquals(Collections.emptyList(), transformers); + } + + @Test + public void testMixedRoutesFile() throws Exception + { + final List expected = prepareSample2(); + + final Resource resource = new ClassPathResource("config/sample2.json"); + final TransformConfigReader loader = TransformConfigReaderFactory.create(resource); + + TransformConfig transformConfig = loader.load(); + final List transformers = transformConfig.getTransformers(); + + assertNotNull(transformers); + assertEquals(expected.size(), transformers.size()); + assertTrue(expected.containsAll(transformers)); + } + + private List prepareSample2() + { + return List.of( + Transformer.builder() + .withTransformerName("CORE_AIO") + .withSupportedSourceAndTargetList(ImmutableSet.of( + SupportedSourceAndTarget.builder() + .withSourceMediaType("image/gif") + .withTargetMediaType("image/gif") + .build() + )) + .withTransformOptions(ImmutableSet.of("imageMagickOptions")) + .build(), + Transformer.builder() + .withTransformerName("IMAGEMAGICK") + .withSupportedSourceAndTargetList(ImmutableSet.of( + SupportedSourceAndTarget.builder() + .withSourceMediaType("image/gif") + .withTargetMediaType("image/gif") + .build() + )) + .withTransformOptions(ImmutableSet.of("imageMagickOptions")) + .build(), + Transformer.builder() + .withTransformerName("CORE_AIO") + .withSupportedSourceAndTargetList(ImmutableSet.of( + SupportedSourceAndTarget.builder() + .withSourceMediaType("application/msword") + .withTargetMediaType("application/pdf") + .withMaxSourceSizeBytes(18874368L) + .build() + )) + .build(), + Transformer.builder() + .withTransformerName("PDF_RENDERER") + .withSupportedSourceAndTargetList(ImmutableSet.of( + SupportedSourceAndTarget.builder() + .withSourceMediaType("application/vnd.ms-powerpoint") + .withTargetMediaType("application/pdf") + .withPriority(55) + .withMaxSourceSizeBytes(50331648L) + .build() + )) + .build(), + Transformer.builder() + .withTransformerName("CORE_AIO") + .withSupportedSourceAndTargetList(ImmutableSet.of( + SupportedSourceAndTarget.builder() + .withSourceMediaType("text/plain") + .withTargetMediaType("text/plain") + .build(), + SupportedSourceAndTarget.builder() + .withSourceMediaType("text/mediawiki") + .withTargetMediaType("text/plain") + .build(), + SupportedSourceAndTarget.builder() + .withSourceMediaType("text/css") + .withTargetMediaType("text/plain") + .build(), + SupportedSourceAndTarget.builder() + .withSourceMediaType("text/csv") + .withTargetMediaType("text/plain") + .build(), + SupportedSourceAndTarget.builder() + .withSourceMediaType("text/xml") + .withTargetMediaType("text/plain") + .build(), + SupportedSourceAndTarget.builder() + .withSourceMediaType("text/html") + .withTargetMediaType("text/plain") + .build(), + SupportedSourceAndTarget.builder() + .withSourceMediaType("application/x-javascript") + .withTargetMediaType("text/plain") + .build(), + SupportedSourceAndTarget.builder() + .withSourceMediaType("application/dita+xml") + .withTargetMediaType("text/plain") + .build() + )) + .withTransformOptions(ImmutableSet.of("stringOptions")) + .build(), + Transformer.builder() + .withTransformerName("officeToImageViaPdf") + .withTransformerPipeline(ImmutableList.of( + new TransformStep("libreoffice", "application/pdf"), + new TransformStep("pdfToImageViaPng", null) + )) + .withTransformOptions(ImmutableSet.of( + "pdfRendererOptions", + "imageMagickOptions" + )) + .build(), + Transformer.builder() + .withTransformerName("textToImageViaPdf") + .withTransformerPipeline(ImmutableList.of( + new TransformStep("libreoffice", "application/pdf"), + new TransformStep("pdfToImageViaPng", null) + )) + .withTransformOptions(ImmutableSet.of( + "pdfRendererOptions", + "imageMagickOptions" + )) + .withSupportedSourceAndTargetList(ImmutableSet.of( + SupportedSourceAndTarget.builder() + .withSourceMediaType("text/plain") + .withTargetMediaType("image/gif") + .build(), + SupportedSourceAndTarget.builder() + .withSourceMediaType("text/plain") + .withTargetMediaType("image/jpeg") + .build(), + SupportedSourceAndTarget.builder() + .withSourceMediaType("text/plain") + .withTargetMediaType("image/tiff") + .build(), + SupportedSourceAndTarget.builder() + .withSourceMediaType("text/plain") + .withTargetMediaType("image/png") + .build(), + SupportedSourceAndTarget.builder() + .withSourceMediaType("text/csv") + .withTargetMediaType("image/gif") + .build(), + SupportedSourceAndTarget.builder() + .withSourceMediaType("text/csv") + .withTargetMediaType("image/jpeg") + .build(), + SupportedSourceAndTarget.builder() + .withSourceMediaType("text/csv") + .withTargetMediaType("image/tiff") + .build(), + SupportedSourceAndTarget.builder() + .withSourceMediaType("text/csv") + .withTargetMediaType("image/png") + .build(), + SupportedSourceAndTarget.builder() + .withSourceMediaType("text/xml") + .withTargetMediaType("image/gif") + .build(), + SupportedSourceAndTarget.builder() + .withSourceMediaType("text/xml") + .withTargetMediaType("image/jpeg") + .build(), + SupportedSourceAndTarget.builder() + .withSourceMediaType("text/xml") + .withTargetMediaType("image/tiff") + .build(), + SupportedSourceAndTarget.builder() + .withSourceMediaType("text/xml") + .withTargetMediaType("image/png") + .build() + )) + .withTransformOptions(ImmutableSet.of( + "pdfRendererOptions", + "imageMagickOptions" + )) + .build() + ); + } + + @Test + public void testRouteFileWithTransformOptions() throws Exception + { + final List expected = prepareSample5Transformers(); + Map> expectedOptions = prepareSample5Options(); + + final Resource resource = new ClassPathResource("config/sample5.json"); + final TransformConfigReader loader = TransformConfigReaderFactory.create(resource); + + TransformConfig transformConfig = loader.load(); + final List transformers = transformConfig.getTransformers(); + Map> transformOptions = transformConfig.getTransformOptions(); + + assertNotNull(transformers); + assertEquals(expected.size(), transformers.size()); + assertTrue(expected.containsAll(transformers)); + assertEquals(expectedOptions, transformOptions); + } + + private List prepareSample5Transformers() + { + return List.of( + Transformer.builder() + .withTransformerName("CORE_AIO") + .withSupportedSourceAndTargetList(ImmutableSet.of( + SupportedSourceAndTarget.builder() + .withSourceMediaType("image/gif") + .withTargetMediaType("image/gif") + .build() + )) + .withTransformOptions(ImmutableSet.of("imageMagickOptions")) + .build() + ); + } + + public static Map> prepareSample5Options() + { + return ImmutableMap.of( + "imageMagickOptions", ImmutableSet.of( + new TransformOptionValue(false, "alphaRemove"), + new TransformOptionValue(false, "autoOrient"), + new TransformOptionValue(false, "startPage"), + new TransformOptionValue(false, "endPage"), + new TransformOptionGroup(false, ImmutableSet.of( + new TransformOptionValue(false, "cropGravity"), + new TransformOptionValue(false, "cropWidth"), + new TransformOptionValue(false, "cropHeight"), + new TransformOptionValue(false, "cropPercentage"), + new TransformOptionValue(false, "cropXOffset"), + new TransformOptionValue(false, "cropYOffset") + )), + new TransformOptionGroup(false, ImmutableSet.of( + new TransformOptionValue(false, "thumbnail"), + new TransformOptionValue(false, "resizeHeight"), + new TransformOptionValue(false, "resizeWidth"), + new TransformOptionValue(false, "resizePercentage"), + new TransformOptionValue(false, "allowEnlargement"), + new TransformOptionValue(false, "maintainAspectRatio") + ))) + ); + } +} \ No newline at end of file diff --git a/model/src/test/java/org/alfresco/transform/config/reader/TransformConfigReaderYamlTest.java b/model/src/test/java/org/alfresco/transform/config/reader/TransformConfigReaderYamlTest.java new file mode 100644 index 00000000..1449d139 --- /dev/null +++ b/model/src/test/java/org/alfresco/transform/config/reader/TransformConfigReaderYamlTest.java @@ -0,0 +1,245 @@ +/* + * #%L + * Alfresco Transform Model + * %% + * Copyright (C) 2015 - 2022 Alfresco Software Limited + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * . + * #L% + */ +package org.alfresco.transform.config.reader; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.alfresco.transform.config.TransformConfig; +import org.alfresco.transform.config.TransformOption; +import org.alfresco.transform.config.Transformer; +import org.alfresco.transform.config.SupportedSourceAndTarget; +import org.alfresco.transform.config.TransformStep; +import org.junit.jupiter.api.Test; +import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.Resource; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +public class TransformConfigReaderYamlTest +{ + @Test + public void testEmptyRoutesFile() throws Exception + { + final Resource resource = new ClassPathResource("config/sample3.yaml"); + final TransformConfigReader loader = TransformConfigReaderFactory.create(resource); + TransformConfig transformConfig = loader.load(); + final List transformers = transformConfig.getTransformers(); + assertNotNull(transformers); + assertEquals(Collections.emptyList(), transformers); + } + + @Test + public void testMixedRoutesFile() throws Exception + { + final List expected = prepareSample4(); + Map> expectedOptions = TransformConfigReaderJsonTest.prepareSample5Options(); + + final Resource resource = new ClassPathResource("config/sample4.yaml"); + final TransformConfigReader loader = TransformConfigReaderFactory.create(resource); + + TransformConfig transformConfig = loader.load(); + final List transformers = transformConfig.getTransformers(); + Map> transformOptions = transformConfig.getTransformOptions(); + + assertNotNull(transformers); + assertEquals(expected.size(), transformers.size()); + assertTrue(expected.containsAll(transformers)); + assertEquals(expectedOptions, transformOptions); + } + + private List prepareSample4() + { + var result = new ArrayList(); + result.add(Transformer.builder() + .withTransformerName("CORE_AIO") + .withSupportedSourceAndTargetList(ImmutableSet.of( + SupportedSourceAndTarget.builder() + .withSourceMediaType("image/gif") + .withTargetMediaType("image/gif") + .build() + )) + .withTransformOptions(ImmutableSet.of("imageMagickOptions")) + .build()); + + result.add(Transformer.builder() + .withTransformerName("IMAGEMAGICK") + .withSupportedSourceAndTargetList(ImmutableSet.of( + SupportedSourceAndTarget.builder() + .withSourceMediaType("image/gif") + .withTargetMediaType("image/gif") + .build() + )) + .withTransformOptions(ImmutableSet.of("imageMagickOptions")) + .build()); + + result.add(Transformer.builder() + .withTransformerName("CORE_AIO") + .withSupportedSourceAndTargetList(ImmutableSet.of( + SupportedSourceAndTarget.builder() + .withSourceMediaType("application/msword") + .withTargetMediaType("application/pdf") + .withMaxSourceSizeBytes(18874368L) + .build() + )) + .build()); + + result.add(Transformer.builder() + .withTransformerName("PDF_RENDERER") + .withSupportedSourceAndTargetList(ImmutableSet.of( + SupportedSourceAndTarget.builder() + .withSourceMediaType("application/vnd.ms-powerpoint") + .withTargetMediaType("application/pdf") + .withPriority(55) + .withMaxSourceSizeBytes(50331648L) + .build() + )) + .build()); + + result.add(Transformer.builder() + .withTransformerName("CORE_AIO") + .withSupportedSourceAndTargetList(ImmutableSet.of( + SupportedSourceAndTarget.builder() + .withSourceMediaType("text/plain") + .withTargetMediaType("text/plain") + .build(), + SupportedSourceAndTarget.builder() + .withSourceMediaType("text/mediawiki") + .withTargetMediaType("text/plain") + .build(), + SupportedSourceAndTarget.builder() + .withSourceMediaType("text/css") + .withTargetMediaType("text/plain") + .build(), + SupportedSourceAndTarget.builder() + .withSourceMediaType("text/csv") + .withTargetMediaType("text/plain") + .build(), + SupportedSourceAndTarget.builder() + .withSourceMediaType("text/xml") + .withTargetMediaType("text/plain") + .build(), + SupportedSourceAndTarget.builder() + .withSourceMediaType("text/html") + .withTargetMediaType("text/plain") + .build(), + SupportedSourceAndTarget.builder() + .withSourceMediaType("application/x-javascript") + .withTargetMediaType("text/plain") + .build(), + SupportedSourceAndTarget.builder() + .withSourceMediaType("application/dita+xml") + .withTargetMediaType("text/plain") + .build() + )) + .withTransformOptions(ImmutableSet.of("stringOptions")) + .build()); + + result.add(Transformer.builder() + .withTransformerName("officeToImageViaPdf") + .withTransformerPipeline(ImmutableList.of( + new TransformStep("libreoffice", "application/pdf"), + new TransformStep("pdfToImageViaPng", null) + )) + .withTransformOptions(ImmutableSet.of( + "pdfRendererOptions", + "imageMagickOptions" + )) + .build()); + + result.add(Transformer.builder() + .withTransformerName("textToImageViaPdf") + .withTransformerPipeline(ImmutableList.of( + new TransformStep("libreoffice", "application/pdf"), + new TransformStep("pdfToImageViaPng", null) + )) + .withTransformOptions(ImmutableSet.of( + "pdfRendererOptions", + "imageMagickOptions" + )) + .withSupportedSourceAndTargetList(ImmutableSet.of( + SupportedSourceAndTarget.builder() + .withSourceMediaType("text/plain") + .withTargetMediaType("image/gif") + .build(), + SupportedSourceAndTarget.builder() + .withSourceMediaType("text/plain") + .withTargetMediaType("image/jpeg") + .build(), + SupportedSourceAndTarget.builder() + .withSourceMediaType("text/plain") + .withTargetMediaType("image/tiff") + .build(), + SupportedSourceAndTarget.builder() + .withSourceMediaType("text/plain") + .withTargetMediaType("image/png") + .build(), + SupportedSourceAndTarget.builder() + .withSourceMediaType("text/csv") + .withTargetMediaType("image/gif") + .build(), + SupportedSourceAndTarget.builder() + .withSourceMediaType("text/csv") + .withTargetMediaType("image/jpeg") + .build(), + SupportedSourceAndTarget.builder() + .withSourceMediaType("text/csv") + .withTargetMediaType("image/tiff") + .build(), + SupportedSourceAndTarget.builder() + .withSourceMediaType("text/csv") + .withTargetMediaType("image/png") + .build(), + SupportedSourceAndTarget.builder() + .withSourceMediaType("text/xml") + .withTargetMediaType("image/gif") + .build(), + SupportedSourceAndTarget.builder() + .withSourceMediaType("text/xml") + .withTargetMediaType("image/jpeg") + .build(), + SupportedSourceAndTarget.builder() + .withSourceMediaType("text/xml") + .withTargetMediaType("image/tiff") + .build(), + SupportedSourceAndTarget.builder() + .withSourceMediaType("text/xml") + .withTargetMediaType("image/png") + .build() + )) + .withTransformOptions(ImmutableSet.of( + "pdfRendererOptions", + "imageMagickOptions" + )) + .build()); + + return result; + } +} diff --git a/model/src/test/java/org/alfresco/transform/registry/TransformRegistryHelperTest.java b/model/src/test/java/org/alfresco/transform/registry/TransformRegistryHelperTest.java index 83895f11..cc05f9cf 100644 --- a/model/src/test/java/org/alfresco/transform/registry/TransformRegistryHelperTest.java +++ b/model/src/test/java/org/alfresco/transform/registry/TransformRegistryHelperTest.java @@ -30,11 +30,13 @@ import com.google.common.collect.ImmutableMap; import org.alfresco.transform.common.TransformException; import org.junit.jupiter.api.Test; +import java.util.HashMap; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import static java.util.Arrays.asList; import static java.util.Collections.emptySet; +import static org.alfresco.transform.common.RequestParamMap.SOURCE_ENCODING; import static org.alfresco.transform.common.RequestParamMap.TIMEOUT; import static org.alfresco.transform.registry.TransformRegistryHelper.retrieveTransformListBySize; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -192,7 +194,21 @@ public class TransformRegistryHelperTest assertThrows(TransformException.class, () -> { - retrieveTransformListBySize(data, "text/plain", null, ImmutableMap.of(TIMEOUT, "1234"), null); + retrieveTransformListBySize(data, "text/plain", null, + new HashMap<>(ImmutableMap.of(TIMEOUT, "1234")), null); + }); + } + + @Test + public void filterSourceEncodingTest() + { + // Almost identical to buildTransformListTargetMimeTypeNullErrorTest + TransformCache data = new TransformCache(); + + assertThrows(TransformException.class, () -> + { + retrieveTransformListBySize(data, "text/plain", null, + new HashMap<>(ImmutableMap.of(SOURCE_ENCODING, "UTF-8")), null); }); } } diff --git a/model/src/test/resources/config/sample1.json b/model/src/test/resources/config/sample1.json new file mode 100644 index 00000000..d3cd00be --- /dev/null +++ b/model/src/test/resources/config/sample1.json @@ -0,0 +1,3 @@ +{ + "transformers":[] +} \ No newline at end of file diff --git a/model/src/test/resources/config/sample2.json b/model/src/test/resources/config/sample2.json new file mode 100644 index 00000000..7f3126c9 --- /dev/null +++ b/model/src/test/resources/config/sample2.json @@ -0,0 +1,92 @@ +{ + "transformers": [ + { + "transformerName": "CORE_AIO", + "supportedSourceAndTargetList": [ + {"sourceMediaType": "image/gif", "targetMediaType": "image/gif" } + ], + "transformOptions": [ + "imageMagickOptions" + ] + }, + { + "transformerName": "IMAGEMAGICK", + "supportedSourceAndTargetList": [ + {"sourceMediaType": "image/gif", "targetMediaType": "image/gif" } + ], + "transformOptions": [ + "imageMagickOptions" + ] + }, + { + "transformerName": "CORE_AIO", + "supportedSourceAndTargetList": [ + {"sourceMediaType": "application/msword", "maxSourceSizeBytes": 18874368, "targetMediaType": "application/pdf" } + ] + }, + { + "transformerName": "PDF_RENDERER", + "supportedSourceAndTargetList": [ + {"sourceMediaType": "application/vnd.ms-powerpoint", "maxSourceSizeBytes": 50331648, "priority": 55, "targetMediaType": "application/pdf" } + ], + "transformOptions": [ + ] + }, + { + "transformerName": "CORE_AIO", + "supportedSourceAndTargetList": [ + {"sourceMediaType": "text/plain", "targetMediaType": "text/plain" }, + {"sourceMediaType": "text/mediawiki", "targetMediaType": "text/plain" }, + {"sourceMediaType": "text/css", "targetMediaType": "text/plain" }, + {"sourceMediaType": "text/csv", "targetMediaType": "text/plain" }, + {"sourceMediaType": "text/xml", "targetMediaType": "text/plain" }, + {"sourceMediaType": "text/html", "targetMediaType": "text/plain" }, + {"sourceMediaType": "application/x-javascript", "targetMediaType": "text/plain" }, + {"sourceMediaType": "application/dita+xml", "targetMediaType": "text/plain" } + ], + "transformOptions": [ + "stringOptions" + ] + }, + { + "transformerName": "officeToImageViaPdf", + "transformerPipeline" : [ + {"transformerName": "libreoffice", "targetMediaType": "application/pdf"}, + {"transformerName": "pdfToImageViaPng"} + ], + "supportedSourceAndTargetList": [ + ], + "transformOptions": [ + "pdfRendererOptions", + "imageMagickOptions" + ] + }, + { + "transformerName": "textToImageViaPdf", + "transformerPipeline" : [ + {"transformerName": "libreoffice", "targetMediaType": "application/pdf"}, + {"transformerName": "pdfToImageViaPng"} + ], + "supportedSourceAndTargetList": [ + {"sourceMediaType": "text/plain", "targetMediaType": "image/gif" }, + {"sourceMediaType": "text/plain", "targetMediaType": "image/jpeg"}, + {"sourceMediaType": "text/plain", "targetMediaType": "image/tiff"}, + {"sourceMediaType": "text/plain", "targetMediaType": "image/png" }, + + {"sourceMediaType": "text/csv", "targetMediaType": "image/gif" }, + {"sourceMediaType": "text/csv", "targetMediaType": "image/jpeg"}, + {"sourceMediaType": "text/csv", "targetMediaType": "image/tiff"}, + {"sourceMediaType": "text/csv", "targetMediaType": "image/png" }, + + {"sourceMediaType": "text/xml", "targetMediaType": "image/gif" }, + {"sourceMediaType": "text/xml", "targetMediaType": "image/jpeg"}, + {"sourceMediaType": "text/xml", "targetMediaType": "image/tiff"}, + {"sourceMediaType": "text/xml", "targetMediaType": "image/png" } + ], + "transformOptions": [ + "pdfRendererOptions", + "imageMagickOptions" + ] + } + ] +} \ No newline at end of file diff --git a/model/src/test/resources/config/sample3.yaml b/model/src/test/resources/config/sample3.yaml new file mode 100644 index 00000000..0f0be92e --- /dev/null +++ b/model/src/test/resources/config/sample3.yaml @@ -0,0 +1,2 @@ +--- +transformers: \ No newline at end of file diff --git a/model/src/test/resources/config/sample4.yaml b/model/src/test/resources/config/sample4.yaml new file mode 100644 index 00000000..22d051f0 --- /dev/null +++ b/model/src/test/resources/config/sample4.yaml @@ -0,0 +1,130 @@ +--- +transformOptions: + imageMagickOptions: + - value: + name: alphaRemove + - value: + name: autoOrient + - value: + name: startPage + - value: + name: endPage + - group: + transformOptions: + - value: + name: cropGravity + - value: + name: cropWidth + - value: + name: cropHeight + - value: + name: cropPercentage + - value: + name: cropXOffset + - value: + name: cropYOffset + - group: + transformOptions: + - value: + name: thumbnail + - value: + name: resizeHeight + - value: + name: resizeWidth + - value: + name: resizePercentage + - value: + name: allowEnlargement + - value: + name: maintainAspectRatio +transformers: + - transformerName: CORE_AIO + supportedSourceAndTargetList: + - sourceMediaType: image/gif + targetMediaType: image/gif + transformOptions: + - imageMagickOptions + + + - transformerName: IMAGEMAGICK + supportedSourceAndTargetList: + - sourceMediaType: image/gif + targetMediaType: image/gif + transformOptions: + - imageMagickOptions + + - transformerName: PDF_RENDERER + supportedSourceAndTargetList: + - sourceMediaType: application/vnd.ms-powerpoint + targetMediaType: application/pdf + priority: 55 + maxSourceSizeBytes: 50331648 + transformOptions: [] + - transformerName: CORE_AIO + supportedSourceAndTargetList: + - sourceMediaType: application/msword + targetMediaType: application/pdf + maxSourceSizeBytes: 18874368 + + - transformerName: CORE_AIO + supportedSourceAndTargetList: + - sourceMediaType: text/plain + targetMediaType: text/plain + - sourceMediaType: text/mediawiki + targetMediaType: text/plain + - sourceMediaType: text/css + targetMediaType: text/plain + - sourceMediaType: text/csv + targetMediaType: text/plain + - sourceMediaType: text/xml + targetMediaType: text/plain + - sourceMediaType: text/html + targetMediaType: text/plain + - sourceMediaType: application/x-javascript + targetMediaType: text/plain + - sourceMediaType: application/dita+xml + targetMediaType: text/plain + transformOptions: + - stringOptions + + - transformerName: officeToImageViaPdf + transformerPipeline: + - transformerName: libreoffice + targetMediaType: application/pdf + - transformerName: pdfToImageViaPng + transformOptions: + - pdfRendererOptions + - imageMagickOptions + - transformerName: textToImageViaPdf + transformerPipeline: + - transformerName: libreoffice + targetMediaType: application/pdf + - transformerName: pdfToImageViaPng + supportedSourceAndTargetList: + - sourceMediaType: text/plain + targetMediaType: image/gif + - sourceMediaType: text/plain + targetMediaType: image/jpeg + - sourceMediaType: text/plain + targetMediaType: image/tiff + - sourceMediaType: text/plain + targetMediaType: image/png + - sourceMediaType: text/csv + targetMediaType: image/gif + - sourceMediaType: text/csv + targetMediaType: image/jpeg + - sourceMediaType: text/csv + targetMediaType: image/tiff + - sourceMediaType: text/csv + targetMediaType: image/png + - sourceMediaType: text/xml + targetMediaType: image/gif + - sourceMediaType: text/xml + targetMediaType: image/jpeg + - sourceMediaType: text/xml + targetMediaType: image/tiff + - sourceMediaType: text/xml + targetMediaType: image/png + transformOptions: + - pdfRendererOptions + - imageMagickOptions diff --git a/model/src/test/resources/config/sample5.json b/model/src/test/resources/config/sample5.json new file mode 100644 index 00000000..4536edce --- /dev/null +++ b/model/src/test/resources/config/sample5.json @@ -0,0 +1,37 @@ +{ + "transformOptions": { + "imageMagickOptions": [ + {"value": {"name": "alphaRemove"}}, + {"value": {"name": "autoOrient"}}, + {"value": {"name": "startPage"}}, + {"value": {"name": "endPage"}}, + {"group": {"transformOptions": [ + {"value": {"name": "cropGravity"}}, + {"value": {"name": "cropWidth"}}, + {"value": {"name": "cropHeight"}}, + {"value": {"name": "cropPercentage"}}, + {"value": {"name": "cropXOffset"}}, + {"value": {"name": "cropYOffset"}} + ]}}, + {"group": {"transformOptions": [ + {"value": {"name": "thumbnail"}}, + {"value": {"name": "resizeHeight"}}, + {"value": {"name": "resizeWidth"}}, + {"value": {"name": "resizePercentage"}}, + {"value": {"name": "allowEnlargement"}}, + {"value": {"name": "maintainAspectRatio"}} + ]}} + ] + }, + "transformers": [ + { + "transformerName": "CORE_AIO", + "supportedSourceAndTargetList": [ + {"sourceMediaType": "image/gif", "targetMediaType": "image/gif" } + ], + "transformOptions": [ + "imageMagickOptions" + ] + } + ] +} \ No newline at end of file diff --git a/pom.xml b/pom.xml index 66a0406d..367fb2f9 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,6 @@ org.springframework.boot spring-boot-starter-parent 2.7.2 -