ACS-2497 T-Core: Accept DAU requests (#525)

* ACS-2497 Add implementation of Direct Access Url usage for transformation. Add possibly to pass Direct Access Url to Transform request instead of a sending a file.
2025-05-12 17:04:48 +00:00 · 2022-02-14 11:30:48 +01:00 · 2022-02-14 11:30:48 +01:00 · 11e3cb4b20
commit 11e3cb4b20
parent f202139607
8 changed files with 211 additions and 26 deletions
--- a/README.md
+++ b/README.md
@ -13,7 +13,7 @@ Contains the common transformer (T-Engine) code, plus a few actual implementatio
 
 ### Documentation

-In addition to the sub-projects (such as `alfresco-transformer-base` README above) some additional documentation can be found in:
+In addition to the subprojects (such as `alfresco-transformer-base` README above) some additional documentation can be found in:

 * [this project's docs](docs) folder
 * [ACS Packaging docs](https://github.com/Alfresco/acs-packaging/tree/master/docs) folder
@ -68,7 +68,7 @@ You can find examples of using Core AIO in the reference ACS Deployment for Dock
 * [ACS Community](https://github.com/Alfresco/acs-deployment/blob/master/docker-compose/community-docker-compose.yml)
 * [ACS Enterprise](https://github.com/Alfresco/acs-deployment/blob/master/docker-compose/docker-compose.yml)

-You can find examples of using the indivudal T-Engines in the reference ACS Deployment for Helm / Kubernetes:
+You can find examples of using the individual T-Engines in the reference ACS Deployment for Helm / Kubernetes:
 * [ACS Community](https://github.com/Alfresco/acs-deployment/blob/master/helm/alfresco-content-services/community_values.yaml)
 * [ACS Enterprise](https://github.com/Alfresco/acs-deployment/blob/master/helm/alfresco-content-services/values.yaml)

--- a/alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/transformer/MiscControllerTest.java
+++ b/alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/transformer/MiscControllerTest.java
@ -532,4 +532,12 @@ public class MiscControllerTest extends AbstractTransformerControllerTest
        text = text.replaceAll("\\n", "");
        return text;
    }
+
+    @Test
+    @Override
+    public void queueTransformRequestUsingDirectAccessUrlTest() throws Exception
+    {
+        super.targetMimetype = this.targetMimetype;
+        super.queueTransformRequestUsingDirectAccessUrlTest();
+    }
 }
--- a/alfresco-transform-tika/alfresco-transform-tika-boot/src/main/resources/templates/transformForm.html
+++ b/alfresco-transform-tika/alfresco-transform-tika-boot/src/main/resources/templates/transformForm.html
@ -6,6 +6,7 @@
        <form method="POST" enctype="multipart/form-data" action="/transform">
            <table>
                <tr><td><div style="text-align:right">file *</div></td><td><input type="file" name="file" /></td></tr>
+                <tr><td><div style="text-align:right">Direct Url</div></td><td><input type="text" name="directAccessUrl"/></td></tr>
                <tr><td><div style="text-align:right">sourceMimetype *</div></td><td><input type="text" name="sourceMimetype" value="application/msword" /></td></tr>
                <tr><td><div style="text-align:right">targetExtension *</div></td><td><input type="text" name="targetExtension" value="txt" /></td></tr>
                <tr><td><div style="text-align:right">targetMimetype *</div></td><td><input type="text" name="targetMimetype" value="text/plain" /></td></tr>
--- a/alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transformer/TikaControllerTest.java
+++ b/alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transformer/TikaControllerTest.java
@ -644,4 +644,15 @@ public class TikaControllerTest extends AbstractTransformerControllerTest
        assertEquals(transformRequest.getClientData(), transformReply.getClientData());
        assertEquals(transformRequest.getSchema(), transformReply.getSchema());
    }
+
+    @Test
+    @Override
+    public void httpTransformRequestUsingDirectAccessUrlTest() throws Exception
+    {
+        this.sourceExtension = PDF;
+        this.targetExtension = TXT;
+        this.sourceMimetype  = MIMETYPE_PDF;
+        expectedTargetFileBytes = readTestFile(targetExtension);
+        super.httpTransformRequestUsingDirectAccessUrlTest();
+    }
 }
--- a/alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transformer/TikaHttpRequestTest.java
+++ b/alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transformer/TikaHttpRequestTest.java
@ -2,7 +2,7 @@
 * #%L
 * Alfresco Transform Core
 * %%
- * Copyright (C) 2005 - 2021 Alfresco Software Limited
+ * Copyright (C) 2005 - 2022 Alfresco Software Limited
 * %%
 * This file is part of the Alfresco software.
 * -
@ -56,18 +56,18 @@ public class TikaHttpRequestTest extends AbstractHttpRequestTest
    // Override method as Tika requires sourceMimetype
    // If not provided then sourceMimetype request parameter error will be thrown.
    @Override
-    protected void assertTransformError(boolean addFile, String errorMessage)
+    protected void assertTransformError(boolean addFile,
+                                        String errorMessage,
+                                        LinkedMultiValueMap<String, Object> additionalParams)
    {
        LinkedMultiValueMap<String, Object> parameters = new LinkedMultiValueMap<>();
-        if (addFile)
-        {
-            parameters.add("file", new ClassPathResource("quick." + getSourceExtension()));
-        }
        parameters.add("sourceMimetype", "application/pdf");
-        HttpHeaders headers = new HttpHeaders();
-        headers.setContentType(MULTIPART_FORM_DATA);
-        HttpEntity<LinkedMultiValueMap<String, Object>> entity = new HttpEntity<>(parameters,
-            headers);
-        super.sendTranformationRequest(entity, errorMessage);
+
+        if (additionalParams != null)
+        {
+            parameters.addAll(additionalParams);
+        }
+
+        super.assertTransformError(addFile, errorMessage, parameters);
    }
 }
--- a/alfresco-transformer-base/src/main/java/org/alfresco/transformer/AbstractTransformerController.java
+++ b/alfresco-transformer-base/src/main/java/org/alfresco/transformer/AbstractTransformerController.java
@ -37,6 +37,7 @@ import org.alfresco.transform.router.TransformerDebug;
 import org.alfresco.transformer.clients.AlfrescoSharedFileStoreClient;
 import org.alfresco.transformer.logging.LogEntry;
 import org.alfresco.transformer.model.FileRefResponse;
+import org.codehaus.plexus.util.FileUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@ -57,6 +58,8 @@ import org.springframework.web.multipart.MultipartFile;

 import javax.servlet.http.HttpServletRequest;
 import java.io.File;
+import java.io.IOException;
+import java.net.URL;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
@ -64,6 +67,7 @@ import java.util.Map;

 import static java.util.stream.Collectors.joining;
 import static org.alfresco.transform.client.model.config.CoreVersionDecorator.setOrClearCoreVersion;
+import static org.alfresco.transform.client.util.RequestParamMap.DIRECT_ACCESS_URL;
 import static org.alfresco.transform.client.util.RequestParamMap.CONFIG_VERSION;
 import static org.alfresco.transform.client.util.RequestParamMap.CONFIG_VERSION_DEFAULT;
 import static org.alfresco.transform.client.util.RequestParamMap.ENDPOINT_TRANSFORM;
@ -130,7 +134,7 @@ public abstract class AbstractTransformerController implements TransformControll

    // Request parameters that are not part of transform options
    public static final List<String> NON_TRANSFORM_OPTION_REQUEST_PARAMETERS = Arrays.asList(SOURCE_EXTENSION,
-            TARGET_EXTENSION, TARGET_MIMETYPE, SOURCE_MIMETYPE, TEST_DELAY, TRANSFORM_NAME_PROPERTY);
+            TARGET_EXTENSION, TARGET_MIMETYPE, SOURCE_MIMETYPE, TEST_DELAY, TRANSFORM_NAME_PROPERTY, DIRECT_ACCESS_URL);

    @Autowired
    private AlfrescoSharedFileStoreClient alfrescoSharedFileStoreClient;
@ -156,7 +160,7 @@ public abstract class AbstractTransformerController implements TransformControll

    @PostMapping(value = ENDPOINT_TRANSFORM, consumes = MULTIPART_FORM_DATA_VALUE)
    public ResponseEntity<Resource> transform(HttpServletRequest request,
-                                              @RequestParam(FILE) MultipartFile sourceMultipartFile,
+                                              @RequestParam(value = FILE, required = false) MultipartFile sourceMultipartFile,
                                              @RequestParam(TARGET_EXTENSION) String targetExtension,
                                              @RequestParam(value = SOURCE_MIMETYPE, required = false) String sourceMimetype,
                                              @RequestParam(value = TARGET_MIMETYPE, required = false) String targetMimetype,
@ -173,10 +177,27 @@ public abstract class AbstractTransformerController implements TransformControll
                    + "targetExtension: '{}', requestParameters: {}", sourceMimetype, targetMimetype, targetExtension, requestParameters);
        }

-        final String targetFilename = createTargetFileName(
-                sourceMultipartFile.getOriginalFilename(), targetExtension);
+        final String directUrl = requestParameters.getOrDefault(DIRECT_ACCESS_URL, "");
+
+        File sourceFile;
+        String sourceFilename;
+        if (directUrl.isBlank())
+        {
+            if (sourceMultipartFile ==  null)
+            {
+                throw new TransformException(BAD_REQUEST.value(), "Required request part 'file' is not present");
+            }
+            sourceFile = createSourceFile(request, sourceMultipartFile);
+            sourceFilename = sourceMultipartFile.getOriginalFilename();
+        }
+        else
+        {
+            sourceFile = getSourceFileFromDirectUrl(directUrl);
+            sourceFilename = sourceFile.getName();
+        }
+
+        final String targetFilename = createTargetFileName(sourceFilename, targetExtension);
        getProbeTestTransform().incrementTransformerCount();
-        final File sourceFile = createSourceFile(request, sourceMultipartFile);
        final File targetFile = createTargetFile(request, targetFilename);

        Map<String, String> transformOptions = getTransformOptions(requestParameters);
@ -191,6 +212,25 @@ public abstract class AbstractTransformerController implements TransformControll
        return body;
    }

+    private File getSourceFileFromDirectUrl(String directUrl)
+    {
+        File sourceFile = createTempFile("tmp", ".tmp");
+        try
+        {
+            FileUtils.copyURLToFile(new URL(directUrl), sourceFile);
+        }
+        catch (IllegalArgumentException e)
+        {
+            throw new TransformException(BAD_REQUEST.value(), "Direct Access Url is invalid.", e);
+        }
+        catch (IOException e)
+        {
+            throw new TransformException(BAD_REQUEST.value(), "Direct Access Url not found.", e);
+        }
+
+        return sourceFile;
+    }
+
    protected Map<String, String> getTransformOptions(Map<String, String> requestParameters)
    {
        Map<String, String> transformOptions = new HashMap<>(requestParameters);
@ -244,9 +284,17 @@ public abstract class AbstractTransformerController implements TransformControll
        // Load the source file
        File sourceFile;
        try
+        {
+            final String directUrl = request.getTransformRequestOptions().getOrDefault(DIRECT_ACCESS_URL, "");
+            if (directUrl.isBlank())
            {
                sourceFile = loadSourceFile(request.getSourceReference(), request.getSourceExtension());
            }
+            else
+            {
+                sourceFile = getSourceFileFromDirectUrl(directUrl);
+            }
+        }
        catch (TransformException e)
        {
            reply.setStatus(e.getStatusCode());
@ -285,7 +333,7 @@ public abstract class AbstractTransformerController implements TransformControll
        {
            String targetMimetype = request.getTargetMediaType();
            String sourceMimetype = request.getSourceMediaType();
-            Map<String, String> transformOptions = request.getTransformRequestOptions();
+            Map<String, String> transformOptions = getTransformOptions(request.getTransformRequestOptions());
            transformerDebug.logOptions(request);
            String transformName = getTransformerName(sourceFile, sourceMimetype, targetMimetype, transformOptions);
            transformImpl(transformName, sourceMimetype, targetMimetype, transformOptions, sourceFile, targetFile);
--- a/alfresco-transformer-base/src/test/java/org/alfresco/transformer/AbstractHttpRequestTest.java
+++ b/alfresco-transformer-base/src/test/java/org/alfresco/transformer/AbstractHttpRequestTest.java
@ -26,6 +26,7 @@
 */
 package org.alfresco.transformer;

+import static org.alfresco.transform.client.util.RequestParamMap.DIRECT_ACCESS_URL;
 import static org.alfresco.transform.client.util.RequestParamMap.ENDPOINT_TRANSFORM;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.springframework.http.HttpMethod.POST;
@ -90,9 +91,12 @@ public abstract class AbstractHttpRequestTest
    @Test
    public void noFileError()
    {
-        // Transformer name is not part of the title as this is checked by another handler
+        LinkedMultiValueMap<String, Object> parameters = new LinkedMultiValueMap<>();
+        parameters.add("targetExtension", ".tmp");
+
        assertTransformError(false,
-            "Required request part 'file' is not present");
+                getTransformerName() + " - Required request part 'file' is not present",
+                parameters);
    }

    @Test
@ -104,10 +108,12 @@ public abstract class AbstractHttpRequestTest
    private void assertMissingParameter(String name)
    {
        assertTransformError(true,
-            getTransformerName() + " - Request parameter '" + name + "' is missing");
+            getTransformerName() + " - Request parameter '" + name + "' is missing", null);
    }

-    protected void assertTransformError(boolean addFile, String errorMessage)
+    protected void assertTransformError(boolean addFile,
+                                        String errorMessage,
+                                        LinkedMultiValueMap<String, Object> additionalParams)
    {
        LinkedMultiValueMap<String, Object> parameters = new LinkedMultiValueMap<>();
        if (addFile)
@ -115,6 +121,11 @@ public abstract class AbstractHttpRequestTest
            parameters.add("file",
                new org.springframework.core.io.ClassPathResource("quick." + getSourceExtension()));
        }
+        if (additionalParams != null)
+        {
+            parameters.addAll(additionalParams);
+        }
+
        HttpHeaders headers = new HttpHeaders();
        headers.setContentType(MULTIPART_FORM_DATA);
        HttpEntity<LinkedMultiValueMap<String, Object>> entity = new HttpEntity<>(parameters,
@ -123,6 +134,21 @@ public abstract class AbstractHttpRequestTest
        sendTranformationRequest(entity, errorMessage);
    }

+    @Test
+    public void httpTransformRequestDirectAccessUrlNotFoundTest()
+    {
+        String directUrl = "https://expired/direct/access/url";
+
+        LinkedMultiValueMap<String, Object> parameters = new LinkedMultiValueMap<>();
+        parameters.add("targetExtension", ".tmp");
+        parameters.add(DIRECT_ACCESS_URL, directUrl);
+
+        assertTransformError(false,
+                getTransformerName() + " - Direct Access Url not found.",
+                parameters);
+
+    }
+
    protected void sendTranformationRequest(
        final HttpEntity<LinkedMultiValueMap<String, Object>> entity, final String errorMessage)
    {
--- a/alfresco-transformer-base/src/test/java/org/alfresco/transformer/AbstractTransformerControllerTest.java
+++ b/alfresco-transformer-base/src/test/java/org/alfresco/transformer/AbstractTransformerControllerTest.java
@ -27,6 +27,7 @@
 package org.alfresco.transformer;

 import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
+import static org.alfresco.transform.client.util.RequestParamMap.DIRECT_ACCESS_URL;
 import static org.alfresco.transform.client.util.RequestParamMap.ENDPOINT_TRANSFORM;
 import static org.alfresco.transform.client.util.RequestParamMap.ENDPOINT_TRANSFORM_CONFIG_LATEST;
 import static org.alfresco.transform.client.util.RequestParamMap.ENDPOINT_TRANSFORM_CONFIG;
@ -34,9 +35,12 @@ import static org.hamcrest.Matchers.containsString;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.when;
 import static org.springframework.http.HttpHeaders.ACCEPT;
 import static org.springframework.http.HttpHeaders.CONTENT_TYPE;
 import static org.springframework.http.HttpStatus.BAD_REQUEST;
+import static org.springframework.http.HttpStatus.CREATED;
 import static org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR;
 import static org.springframework.http.HttpStatus.OK;
 import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE;
@ -55,6 +59,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.UUID;

 import org.alfresco.transform.client.model.InternalContext;
 import org.alfresco.transform.client.model.TransformReply;
@ -68,6 +73,8 @@ import org.alfresco.transform.client.model.config.Transformer;
 import org.alfresco.transform.client.registry.TransformServiceRegistry;
 import org.alfresco.transform.router.TransformStack;
 import org.alfresco.transformer.clients.AlfrescoSharedFileStoreClient;
+import org.alfresco.transformer.model.FileRefEntity;
+import org.alfresco.transformer.model.FileRefResponse;
 import org.alfresco.transformer.probes.ProbeTestTransform;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
@ -215,6 +222,36 @@ public abstract class AbstractTransformerControllerTest

    protected MockHttpServletRequestBuilder mockMvcRequest(String url, MockMultipartFile sourceFile,
                                                           String... params)
+    {
+        if (sourceFile == null)
+        {
+            return mockMvcRequestWithoutMockMultipartFile(url, params);
+        }
+        else
+        {
+            return mockMvcRequestWithMockMultipartFile(url, sourceFile, params);
+        }
+    }
+
+    private MockHttpServletRequestBuilder mockMvcRequestWithoutMockMultipartFile(String url,
+                                                           String... params)
+    {
+        MockHttpServletRequestBuilder builder = MockMvcRequestBuilders.multipart(ENDPOINT_TRANSFORM);
+
+        if (params.length % 2 != 0)
+        {
+            throw new IllegalArgumentException("each param should have a name and value.");
+        }
+        for (int i = 0; i < params.length; i += 2)
+        {
+            builder = builder.param(params[i], params[i + 1]);
+        }
+
+        return builder;
+    }
+
+    private MockHttpServletRequestBuilder mockMvcRequestWithMockMultipartFile(String url, MockMultipartFile sourceFile,
+        String... params)
    {
        MockHttpServletRequestBuilder builder = MockMvcRequestBuilders.multipart(ENDPOINT_TRANSFORM).file(
            sourceFile);
@ -569,4 +606,58 @@ public abstract class AbstractTransformerControllerTest
        transformer.setSupportedSourceAndTargetList(supportedSourceAndTargetList);
        return transformer;
    }
+
+    @Test
+    public void queueTransformRequestUsingDirectAccessUrlTest() throws Exception
+    {
+        // Files
+        String sourceFileRef = UUID.randomUUID().toString();
+        File sourceFile = getTestFile("quick." + sourceExtension, true);
+        String targetFileRef = UUID.randomUUID().toString();
+
+        TransformRequest transformRequest = createTransformRequest(sourceFileRef, sourceFile);
+        Map<String, String> transformRequestOptions = transformRequest.getTransformRequestOptions();
+
+        String directUrl = "file://" + sourceFile.toPath();
+
+        transformRequestOptions.put(DIRECT_ACCESS_URL, directUrl);
+        transformRequest.setTransformRequestOptions(transformRequestOptions);
+
+        when(alfrescoSharedFileStoreClient.saveFile(any()))
+                .thenReturn(new FileRefResponse(new FileRefEntity(targetFileRef)));
+
+        // Update the Transformation Request with any specific params before sending it
+        updateTransformRequestWithSpecificOptions(transformRequest);
+
+        // Serialize and call the transformer
+        String tr = objectMapper.writeValueAsString(transformRequest);
+        String transformationReplyAsString = mockMvc
+                .perform(MockMvcRequestBuilders
+                        .post("/transform")
+                        .header(ACCEPT, APPLICATION_JSON_VALUE)
+                        .header(CONTENT_TYPE, APPLICATION_JSON_VALUE)
+                        .content(tr))
+                .andExpect(status().is(CREATED.value()))
+                .andReturn().getResponse().getContentAsString();
+
+        TransformReply transformReply = objectMapper.readValue(transformationReplyAsString,
+                TransformReply.class);
+
+        // Assert the reply
+        assertEquals(transformRequest.getRequestId(), transformReply.getRequestId());
+        assertEquals(transformRequest.getClientData(), transformReply.getClientData());
+        assertEquals(transformRequest.getSchema(), transformReply.getSchema());
+    }
+
+    @Test
+    public void httpTransformRequestUsingDirectAccessUrlTest() throws Exception
+    {
+        File dauSourceFile = getTestFile("quick." + sourceExtension, true);
+        String directUrl = "file://" + dauSourceFile.toPath();
+
+        mockMvc.perform(
+                       mockMvcRequest("/transform", null, "targetExtension", targetExtension, DIRECT_ACCESS_URL, directUrl))
+               .andExpect(status().is(OK.value()))
+               .andExpect(content().bytes(expectedTargetFileBytes));
+    }
 }