From 11e3cb4b205b75c5564ac3a1736f3f5cd144c9d0 Mon Sep 17 00:00:00 2001 From: Kacper Magdziarz <95610011+kmagdziarz@users.noreply.github.com> Date: Mon, 14 Feb 2022 11:30:48 +0100 Subject: [PATCH] ACS-2497 T-Core: Accept DAU requests (#525) * ACS-2497 Add implementation of Direct Access Url usage for transformation. Add possibly to pass Direct Access Url to Transform request instead of a sending a file. --- README.md | 4 +- .../transformer/MiscControllerTest.java | 8 ++ .../resources/templates/transformForm.html | 1 + .../transformer/TikaControllerTest.java | 11 +++ .../transformer/TikaHttpRequestTest.java | 22 ++--- .../AbstractTransformerController.java | 66 ++++++++++++-- .../transformer/AbstractHttpRequestTest.java | 34 ++++++- .../AbstractTransformerControllerTest.java | 91 +++++++++++++++++++ 8 files changed, 211 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 2093f3e2..4723c2b2 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Contains the common transformer (T-Engine) code, plus a few actual implementatio ### Documentation -In addition to the sub-projects (such as `alfresco-transformer-base` README above) some additional documentation can be found in: +In addition to the subprojects (such as `alfresco-transformer-base` README above) some additional documentation can be found in: * [this project's docs](docs) folder * [ACS Packaging docs](https://github.com/Alfresco/acs-packaging/tree/master/docs) folder @@ -68,7 +68,7 @@ You can find examples of using Core AIO in the reference ACS Deployment for Dock * [ACS Community](https://github.com/Alfresco/acs-deployment/blob/master/docker-compose/community-docker-compose.yml) * [ACS Enterprise](https://github.com/Alfresco/acs-deployment/blob/master/docker-compose/docker-compose.yml) -You can find examples of using the indivudal T-Engines in the reference ACS Deployment for Helm / Kubernetes: +You can find examples of using the individual T-Engines in the reference ACS Deployment for Helm / Kubernetes: * [ACS Community](https://github.com/Alfresco/acs-deployment/blob/master/helm/alfresco-content-services/community_values.yaml) * [ACS Enterprise](https://github.com/Alfresco/acs-deployment/blob/master/helm/alfresco-content-services/values.yaml) diff --git a/alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/transformer/MiscControllerTest.java b/alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/transformer/MiscControllerTest.java index 89c1f71b..e4dec7e0 100644 --- a/alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/transformer/MiscControllerTest.java +++ b/alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/transformer/MiscControllerTest.java @@ -532,4 +532,12 @@ public class MiscControllerTest extends AbstractTransformerControllerTest text = text.replaceAll("\\n", ""); return text; } + + @Test + @Override + public void queueTransformRequestUsingDirectAccessUrlTest() throws Exception + { + super.targetMimetype = this.targetMimetype; + super.queueTransformRequestUsingDirectAccessUrlTest(); + } } \ No newline at end of file diff --git a/alfresco-transform-tika/alfresco-transform-tika-boot/src/main/resources/templates/transformForm.html b/alfresco-transform-tika/alfresco-transform-tika-boot/src/main/resources/templates/transformForm.html index d6d9ba76..6d47ff5f 100644 --- a/alfresco-transform-tika/alfresco-transform-tika-boot/src/main/resources/templates/transformForm.html +++ b/alfresco-transform-tika/alfresco-transform-tika-boot/src/main/resources/templates/transformForm.html @@ -6,6 +6,7 @@
+ diff --git a/alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transformer/TikaControllerTest.java b/alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transformer/TikaControllerTest.java index c379e567..83541ec8 100644 --- a/alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transformer/TikaControllerTest.java +++ b/alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transformer/TikaControllerTest.java @@ -644,4 +644,15 @@ public class TikaControllerTest extends AbstractTransformerControllerTest assertEquals(transformRequest.getClientData(), transformReply.getClientData()); assertEquals(transformRequest.getSchema(), transformReply.getSchema()); } + + @Test + @Override + public void httpTransformRequestUsingDirectAccessUrlTest() throws Exception + { + this.sourceExtension = PDF; + this.targetExtension = TXT; + this.sourceMimetype = MIMETYPE_PDF; + expectedTargetFileBytes = readTestFile(targetExtension); + super.httpTransformRequestUsingDirectAccessUrlTest(); + } } diff --git a/alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transformer/TikaHttpRequestTest.java b/alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transformer/TikaHttpRequestTest.java index c972f3b2..56aef563 100644 --- a/alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transformer/TikaHttpRequestTest.java +++ b/alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transformer/TikaHttpRequestTest.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2021 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -56,18 +56,18 @@ public class TikaHttpRequestTest extends AbstractHttpRequestTest // Override method as Tika requires sourceMimetype // If not provided then sourceMimetype request parameter error will be thrown. @Override - protected void assertTransformError(boolean addFile, String errorMessage) + protected void assertTransformError(boolean addFile, + String errorMessage, + LinkedMultiValueMap additionalParams) { LinkedMultiValueMap parameters = new LinkedMultiValueMap<>(); - if (addFile) - { - parameters.add("file", new ClassPathResource("quick." + getSourceExtension())); - } parameters.add("sourceMimetype", "application/pdf"); - HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MULTIPART_FORM_DATA); - HttpEntity> entity = new HttpEntity<>(parameters, - headers); - super.sendTranformationRequest(entity, errorMessage); + + if (additionalParams != null) + { + parameters.addAll(additionalParams); + } + + super.assertTransformError(addFile, errorMessage, parameters); } } diff --git a/alfresco-transformer-base/src/main/java/org/alfresco/transformer/AbstractTransformerController.java b/alfresco-transformer-base/src/main/java/org/alfresco/transformer/AbstractTransformerController.java index af22c2ad..a3ffcf3c 100644 --- a/alfresco-transformer-base/src/main/java/org/alfresco/transformer/AbstractTransformerController.java +++ b/alfresco-transformer-base/src/main/java/org/alfresco/transformer/AbstractTransformerController.java @@ -37,6 +37,7 @@ import org.alfresco.transform.router.TransformerDebug; import org.alfresco.transformer.clients.AlfrescoSharedFileStoreClient; import org.alfresco.transformer.logging.LogEntry; import org.alfresco.transformer.model.FileRefResponse; +import org.codehaus.plexus.util.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -57,6 +58,8 @@ import org.springframework.web.multipart.MultipartFile; import javax.servlet.http.HttpServletRequest; import java.io.File; +import java.io.IOException; +import java.net.URL; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -64,6 +67,7 @@ import java.util.Map; import static java.util.stream.Collectors.joining; import static org.alfresco.transform.client.model.config.CoreVersionDecorator.setOrClearCoreVersion; +import static org.alfresco.transform.client.util.RequestParamMap.DIRECT_ACCESS_URL; import static org.alfresco.transform.client.util.RequestParamMap.CONFIG_VERSION; import static org.alfresco.transform.client.util.RequestParamMap.CONFIG_VERSION_DEFAULT; import static org.alfresco.transform.client.util.RequestParamMap.ENDPOINT_TRANSFORM; @@ -130,7 +134,7 @@ public abstract class AbstractTransformerController implements TransformControll // Request parameters that are not part of transform options public static final List NON_TRANSFORM_OPTION_REQUEST_PARAMETERS = Arrays.asList(SOURCE_EXTENSION, - TARGET_EXTENSION, TARGET_MIMETYPE, SOURCE_MIMETYPE, TEST_DELAY, TRANSFORM_NAME_PROPERTY); + TARGET_EXTENSION, TARGET_MIMETYPE, SOURCE_MIMETYPE, TEST_DELAY, TRANSFORM_NAME_PROPERTY, DIRECT_ACCESS_URL); @Autowired private AlfrescoSharedFileStoreClient alfrescoSharedFileStoreClient; @@ -156,16 +160,16 @@ public abstract class AbstractTransformerController implements TransformControll @PostMapping(value = ENDPOINT_TRANSFORM, consumes = MULTIPART_FORM_DATA_VALUE) public ResponseEntity transform(HttpServletRequest request, - @RequestParam(FILE) MultipartFile sourceMultipartFile, + @RequestParam(value = FILE, required = false) MultipartFile sourceMultipartFile, @RequestParam(TARGET_EXTENSION) String targetExtension, @RequestParam(value = SOURCE_MIMETYPE, required = false) String sourceMimetype, @RequestParam(value = TARGET_MIMETYPE, required = false) String targetMimetype, @RequestParam Map requestParameters, - @RequestParam (value = TEST_DELAY, required = false) Long testDelay, + @RequestParam(value = TEST_DELAY, required = false) Long testDelay, // The TRANSFORM_NAME_PROPERTY param allows ACS legacy transformers to specify which transform to use, // It can be removed once legacy transformers are removed from ACS. - @RequestParam (value = TRANSFORM_NAME_PROPERTY, required = false) String requestTransformName) + @RequestParam(value = TRANSFORM_NAME_PROPERTY, required = false) String requestTransformName) { if (logger.isDebugEnabled()) { @@ -173,10 +177,27 @@ public abstract class AbstractTransformerController implements TransformControll + "targetExtension: '{}', requestParameters: {}", sourceMimetype, targetMimetype, targetExtension, requestParameters); } - final String targetFilename = createTargetFileName( - sourceMultipartFile.getOriginalFilename(), targetExtension); + final String directUrl = requestParameters.getOrDefault(DIRECT_ACCESS_URL, ""); + + File sourceFile; + String sourceFilename; + if (directUrl.isBlank()) + { + if (sourceMultipartFile == null) + { + throw new TransformException(BAD_REQUEST.value(), "Required request part 'file' is not present"); + } + sourceFile = createSourceFile(request, sourceMultipartFile); + sourceFilename = sourceMultipartFile.getOriginalFilename(); + } + else + { + sourceFile = getSourceFileFromDirectUrl(directUrl); + sourceFilename = sourceFile.getName(); + } + + final String targetFilename = createTargetFileName(sourceFilename, targetExtension); getProbeTestTransform().incrementTransformerCount(); - final File sourceFile = createSourceFile(request, sourceMultipartFile); final File targetFile = createTargetFile(request, targetFilename); Map transformOptions = getTransformOptions(requestParameters); @@ -191,6 +212,25 @@ public abstract class AbstractTransformerController implements TransformControll return body; } + private File getSourceFileFromDirectUrl(String directUrl) + { + File sourceFile = createTempFile("tmp", ".tmp"); + try + { + FileUtils.copyURLToFile(new URL(directUrl), sourceFile); + } + catch (IllegalArgumentException e) + { + throw new TransformException(BAD_REQUEST.value(), "Direct Access Url is invalid.", e); + } + catch (IOException e) + { + throw new TransformException(BAD_REQUEST.value(), "Direct Access Url not found.", e); + } + + return sourceFile; + } + protected Map getTransformOptions(Map requestParameters) { Map transformOptions = new HashMap<>(requestParameters); @@ -245,7 +285,15 @@ public abstract class AbstractTransformerController implements TransformControll File sourceFile; try { - sourceFile = loadSourceFile(request.getSourceReference(), request.getSourceExtension()); + final String directUrl = request.getTransformRequestOptions().getOrDefault(DIRECT_ACCESS_URL, ""); + if (directUrl.isBlank()) + { + sourceFile = loadSourceFile(request.getSourceReference(), request.getSourceExtension()); + } + else + { + sourceFile = getSourceFileFromDirectUrl(directUrl); + } } catch (TransformException e) { @@ -285,7 +333,7 @@ public abstract class AbstractTransformerController implements TransformControll { String targetMimetype = request.getTargetMediaType(); String sourceMimetype = request.getSourceMediaType(); - Map transformOptions = request.getTransformRequestOptions(); + Map transformOptions = getTransformOptions(request.getTransformRequestOptions()); transformerDebug.logOptions(request); String transformName = getTransformerName(sourceFile, sourceMimetype, targetMimetype, transformOptions); transformImpl(transformName, sourceMimetype, targetMimetype, transformOptions, sourceFile, targetFile); diff --git a/alfresco-transformer-base/src/test/java/org/alfresco/transformer/AbstractHttpRequestTest.java b/alfresco-transformer-base/src/test/java/org/alfresco/transformer/AbstractHttpRequestTest.java index 7bafa372..6dd6c245 100644 --- a/alfresco-transformer-base/src/test/java/org/alfresco/transformer/AbstractHttpRequestTest.java +++ b/alfresco-transformer-base/src/test/java/org/alfresco/transformer/AbstractHttpRequestTest.java @@ -26,6 +26,7 @@ */ package org.alfresco.transformer; +import static org.alfresco.transform.client.util.RequestParamMap.DIRECT_ACCESS_URL; import static org.alfresco.transform.client.util.RequestParamMap.ENDPOINT_TRANSFORM; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.springframework.http.HttpMethod.POST; @@ -90,9 +91,12 @@ public abstract class AbstractHttpRequestTest @Test public void noFileError() { - // Transformer name is not part of the title as this is checked by another handler + LinkedMultiValueMap parameters = new LinkedMultiValueMap<>(); + parameters.add("targetExtension", ".tmp"); + assertTransformError(false, - "Required request part 'file' is not present"); + getTransformerName() + " - Required request part 'file' is not present", + parameters); } @Test @@ -104,10 +108,12 @@ public abstract class AbstractHttpRequestTest private void assertMissingParameter(String name) { assertTransformError(true, - getTransformerName() + " - Request parameter '" + name + "' is missing"); + getTransformerName() + " - Request parameter '" + name + "' is missing", null); } - protected void assertTransformError(boolean addFile, String errorMessage) + protected void assertTransformError(boolean addFile, + String errorMessage, + LinkedMultiValueMap additionalParams) { LinkedMultiValueMap parameters = new LinkedMultiValueMap<>(); if (addFile) @@ -115,6 +121,11 @@ public abstract class AbstractHttpRequestTest parameters.add("file", new org.springframework.core.io.ClassPathResource("quick." + getSourceExtension())); } + if (additionalParams != null) + { + parameters.addAll(additionalParams); + } + HttpHeaders headers = new HttpHeaders(); headers.setContentType(MULTIPART_FORM_DATA); HttpEntity> entity = new HttpEntity<>(parameters, @@ -123,6 +134,21 @@ public abstract class AbstractHttpRequestTest sendTranformationRequest(entity, errorMessage); } + @Test + public void httpTransformRequestDirectAccessUrlNotFoundTest() + { + String directUrl = "https://expired/direct/access/url"; + + LinkedMultiValueMap parameters = new LinkedMultiValueMap<>(); + parameters.add("targetExtension", ".tmp"); + parameters.add(DIRECT_ACCESS_URL, directUrl); + + assertTransformError(false, + getTransformerName() + " - Direct Access Url not found.", + parameters); + + } + protected void sendTranformationRequest( final HttpEntity> entity, final String errorMessage) { diff --git a/alfresco-transformer-base/src/test/java/org/alfresco/transformer/AbstractTransformerControllerTest.java b/alfresco-transformer-base/src/test/java/org/alfresco/transformer/AbstractTransformerControllerTest.java index 77ce416d..616119e2 100644 --- a/alfresco-transformer-base/src/test/java/org/alfresco/transformer/AbstractTransformerControllerTest.java +++ b/alfresco-transformer-base/src/test/java/org/alfresco/transformer/AbstractTransformerControllerTest.java @@ -27,6 +27,7 @@ package org.alfresco.transformer; import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; +import static org.alfresco.transform.client.util.RequestParamMap.DIRECT_ACCESS_URL; import static org.alfresco.transform.client.util.RequestParamMap.ENDPOINT_TRANSFORM; import static org.alfresco.transform.client.util.RequestParamMap.ENDPOINT_TRANSFORM_CONFIG_LATEST; import static org.alfresco.transform.client.util.RequestParamMap.ENDPOINT_TRANSFORM_CONFIG; @@ -34,9 +35,12 @@ import static org.hamcrest.Matchers.containsString; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; import static org.springframework.http.HttpHeaders.ACCEPT; import static org.springframework.http.HttpHeaders.CONTENT_TYPE; import static org.springframework.http.HttpStatus.BAD_REQUEST; +import static org.springframework.http.HttpStatus.CREATED; import static org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR; import static org.springframework.http.HttpStatus.OK; import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; @@ -55,6 +59,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.UUID; import org.alfresco.transform.client.model.InternalContext; import org.alfresco.transform.client.model.TransformReply; @@ -68,6 +73,8 @@ import org.alfresco.transform.client.model.config.Transformer; import org.alfresco.transform.client.registry.TransformServiceRegistry; import org.alfresco.transform.router.TransformStack; import org.alfresco.transformer.clients.AlfrescoSharedFileStoreClient; +import org.alfresco.transformer.model.FileRefEntity; +import org.alfresco.transformer.model.FileRefResponse; import org.alfresco.transformer.probes.ProbeTestTransform; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -214,6 +221,36 @@ public abstract class AbstractTransformerControllerTest } protected MockHttpServletRequestBuilder mockMvcRequest(String url, MockMultipartFile sourceFile, + String... params) + { + if (sourceFile == null) + { + return mockMvcRequestWithoutMockMultipartFile(url, params); + } + else + { + return mockMvcRequestWithMockMultipartFile(url, sourceFile, params); + } + } + + private MockHttpServletRequestBuilder mockMvcRequestWithoutMockMultipartFile(String url, + String... params) + { + MockHttpServletRequestBuilder builder = MockMvcRequestBuilders.multipart(ENDPOINT_TRANSFORM); + + if (params.length % 2 != 0) + { + throw new IllegalArgumentException("each param should have a name and value."); + } + for (int i = 0; i < params.length; i += 2) + { + builder = builder.param(params[i], params[i + 1]); + } + + return builder; + } + + private MockHttpServletRequestBuilder mockMvcRequestWithMockMultipartFile(String url, MockMultipartFile sourceFile, String... params) { MockHttpServletRequestBuilder builder = MockMvcRequestBuilders.multipart(ENDPOINT_TRANSFORM).file( @@ -569,4 +606,58 @@ public abstract class AbstractTransformerControllerTest transformer.setSupportedSourceAndTargetList(supportedSourceAndTargetList); return transformer; } + + @Test + public void queueTransformRequestUsingDirectAccessUrlTest() throws Exception + { + // Files + String sourceFileRef = UUID.randomUUID().toString(); + File sourceFile = getTestFile("quick." + sourceExtension, true); + String targetFileRef = UUID.randomUUID().toString(); + + TransformRequest transformRequest = createTransformRequest(sourceFileRef, sourceFile); + Map transformRequestOptions = transformRequest.getTransformRequestOptions(); + + String directUrl = "file://" + sourceFile.toPath(); + + transformRequestOptions.put(DIRECT_ACCESS_URL, directUrl); + transformRequest.setTransformRequestOptions(transformRequestOptions); + + when(alfrescoSharedFileStoreClient.saveFile(any())) + .thenReturn(new FileRefResponse(new FileRefEntity(targetFileRef))); + + // Update the Transformation Request with any specific params before sending it + updateTransformRequestWithSpecificOptions(transformRequest); + + // Serialize and call the transformer + String tr = objectMapper.writeValueAsString(transformRequest); + String transformationReplyAsString = mockMvc + .perform(MockMvcRequestBuilders + .post("/transform") + .header(ACCEPT, APPLICATION_JSON_VALUE) + .header(CONTENT_TYPE, APPLICATION_JSON_VALUE) + .content(tr)) + .andExpect(status().is(CREATED.value())) + .andReturn().getResponse().getContentAsString(); + + TransformReply transformReply = objectMapper.readValue(transformationReplyAsString, + TransformReply.class); + + // Assert the reply + assertEquals(transformRequest.getRequestId(), transformReply.getRequestId()); + assertEquals(transformRequest.getClientData(), transformReply.getClientData()); + assertEquals(transformRequest.getSchema(), transformReply.getSchema()); + } + + @Test + public void httpTransformRequestUsingDirectAccessUrlTest() throws Exception + { + File dauSourceFile = getTestFile("quick." + sourceExtension, true); + String directUrl = "file://" + dauSourceFile.toPath(); + + mockMvc.perform( + mockMvcRequest("/transform", null, "targetExtension", targetExtension, DIRECT_ACCESS_URL, directUrl)) + .andExpect(status().is(OK.value())) + .andExpect(content().bytes(expectedTargetFileBytes)); + } }
file *
Direct Url
sourceMimetype *
targetExtension *
targetMimetype *