ACS-2497 T-Core: Accept DAU requests (#525)

* ACS-2497 Add implementation of Direct Access Url usage for transformation.
Add possibly to pass Direct Access Url to Transform request instead of a sending a file.

README.md

@@ -13,7 +13,7 @@ Contains the common transformer (T-Engine) code, plus a few actual implementatio
  
 ### Documentation
 
-In addition to the sub-projects (such as `alfresco-transformer-base` README above) some additional documentation can be found in:
+In addition to the subprojects (such as `alfresco-transformer-base` README above) some additional documentation can be found in:
 
 * [this project's docs](docs) folder
 * [ACS Packaging docs](https://github.com/Alfresco/acs-packaging/tree/master/docs) folder
@@ -68,7 +68,7 @@ You can find examples of using Core AIO in the reference ACS Deployment for Dock
 * [ACS Community](https://github.com/Alfresco/acs-deployment/blob/master/docker-compose/community-docker-compose.yml)
 * [ACS Enterprise](https://github.com/Alfresco/acs-deployment/blob/master/docker-compose/docker-compose.yml)
 
-You can find examples of using the indivudal T-Engines in the reference ACS Deployment for Helm / Kubernetes:
+You can find examples of using the individual T-Engines in the reference ACS Deployment for Helm / Kubernetes:
 * [ACS Community](https://github.com/Alfresco/acs-deployment/blob/master/helm/alfresco-content-services/community_values.yaml)
 * [ACS Enterprise](https://github.com/Alfresco/acs-deployment/blob/master/helm/alfresco-content-services/values.yaml)
 

alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/transformer/MiscControllerTest.java

@@ -532,4 +532,12 @@ public class MiscControllerTest extends AbstractTransformerControllerTest
         text = text.replaceAll("\\n", "");
         return text;
     }
+
+    @Test
+    @Override
+    public void queueTransformRequestUsingDirectAccessUrlTest() throws Exception
+    {
+        super.targetMimetype = this.targetMimetype;
+        super.queueTransformRequestUsingDirectAccessUrlTest();
+    }
 }

alfresco-transform-tika/alfresco-transform-tika-boot/src/main/resources/templates/transformForm.html

@@ -6,6 +6,7 @@
         <form method="POST" enctype="multipart/form-data" action="/transform">
             <table>
                 <tr><td><div style="text-align:right">file *</div></td><td><input type="file" name="file" /></td></tr>
+                <tr><td><div style="text-align:right">Direct Url</div></td><td><input type="text" name="directAccessUrl"/></td></tr>
                 <tr><td><div style="text-align:right">sourceMimetype *</div></td><td><input type="text" name="sourceMimetype" value="application/msword" /></td></tr>
                 <tr><td><div style="text-align:right">targetExtension *</div></td><td><input type="text" name="targetExtension" value="txt" /></td></tr>
                 <tr><td><div style="text-align:right">targetMimetype *</div></td><td><input type="text" name="targetMimetype" value="text/plain" /></td></tr>

alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transformer/TikaControllerTest.java

@@ -644,4 +644,15 @@ public class TikaControllerTest extends AbstractTransformerControllerTest
         assertEquals(transformRequest.getClientData(), transformReply.getClientData());
         assertEquals(transformRequest.getSchema(), transformReply.getSchema());
     }
+
+    @Test
+    @Override
+    public void httpTransformRequestUsingDirectAccessUrlTest() throws Exception
+    {
+        this.sourceExtension = PDF;
+        this.targetExtension = TXT;
+        this.sourceMimetype  = MIMETYPE_PDF;
+        expectedTargetFileBytes = readTestFile(targetExtension);
+        super.httpTransformRequestUsingDirectAccessUrlTest();
+    }
 }

alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transformer/TikaHttpRequestTest.java

@@ -2,7 +2,7 @@
  * #%L
  * Alfresco Transform Core
  * %%
- * Copyright (C) 2005 - 2021 Alfresco Software Limited
+ * Copyright (C) 2005 - 2022 Alfresco Software Limited
  * %%
  * This file is part of the Alfresco software.
  * -
@@ -56,18 +56,18 @@ public class TikaHttpRequestTest extends AbstractHttpRequestTest
     // Override method as Tika requires sourceMimetype
     // If not provided then sourceMimetype request parameter error will be thrown.
     @Override
-    protected void assertTransformError(boolean addFile, String errorMessage)
+    protected void assertTransformError(boolean addFile,
+                                        String errorMessage,
+                                        LinkedMultiValueMap<String, Object> additionalParams)
     {
         LinkedMultiValueMap<String, Object> parameters = new LinkedMultiValueMap<>();
-        if (addFile)
-        {
-            parameters.add("file", new ClassPathResource("quick." + getSourceExtension()));
-        }
         parameters.add("sourceMimetype", "application/pdf");
-        HttpHeaders headers = new HttpHeaders();
+
-        headers.setContentType(MULTIPART_FORM_DATA);
+        if (additionalParams != null)
-        HttpEntity<LinkedMultiValueMap<String, Object>> entity = new HttpEntity<>(parameters,
+        {
-            headers);
+            parameters.addAll(additionalParams);
-        super.sendTranformationRequest(entity, errorMessage);
+        }
+
+        super.assertTransformError(addFile, errorMessage, parameters);
     }
 }

alfresco-transformer-base/src/main/java/org/alfresco/transformer/AbstractTransformerController.java

@@ -37,6 +37,7 @@ import org.alfresco.transform.router.TransformerDebug;
 import org.alfresco.transformer.clients.AlfrescoSharedFileStoreClient;
 import org.alfresco.transformer.logging.LogEntry;
 import org.alfresco.transformer.model.FileRefResponse;
+import org.codehaus.plexus.util.FileUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -57,6 +58,8 @@ import org.springframework.web.multipart.MultipartFile;
 
 import javax.servlet.http.HttpServletRequest;
 import java.io.File;
+import java.io.IOException;
+import java.net.URL;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
@@ -64,6 +67,7 @@ import java.util.Map;
 
 import static java.util.stream.Collectors.joining;
 import static org.alfresco.transform.client.model.config.CoreVersionDecorator.setOrClearCoreVersion;
+import static org.alfresco.transform.client.util.RequestParamMap.DIRECT_ACCESS_URL;
 import static org.alfresco.transform.client.util.RequestParamMap.CONFIG_VERSION;
 import static org.alfresco.transform.client.util.RequestParamMap.CONFIG_VERSION_DEFAULT;
 import static org.alfresco.transform.client.util.RequestParamMap.ENDPOINT_TRANSFORM;
@@ -130,7 +134,7 @@ public abstract class AbstractTransformerController implements TransformControll
 
     // Request parameters that are not part of transform options
     public static final List<String> NON_TRANSFORM_OPTION_REQUEST_PARAMETERS = Arrays.asList(SOURCE_EXTENSION,
-            TARGET_EXTENSION, TARGET_MIMETYPE, SOURCE_MIMETYPE, TEST_DELAY, TRANSFORM_NAME_PROPERTY);
+            TARGET_EXTENSION, TARGET_MIMETYPE, SOURCE_MIMETYPE, TEST_DELAY, TRANSFORM_NAME_PROPERTY, DIRECT_ACCESS_URL);
 
     @Autowired
     private AlfrescoSharedFileStoreClient alfrescoSharedFileStoreClient;
@@ -156,16 +160,16 @@ public abstract class AbstractTransformerController implements TransformControll
 
     @PostMapping(value = ENDPOINT_TRANSFORM, consumes = MULTIPART_FORM_DATA_VALUE)
     public ResponseEntity<Resource> transform(HttpServletRequest request,
-                                              @RequestParam(FILE) MultipartFile sourceMultipartFile,
+                                              @RequestParam(value = FILE, required = false) MultipartFile sourceMultipartFile,
                                               @RequestParam(TARGET_EXTENSION) String targetExtension,
                                               @RequestParam(value = SOURCE_MIMETYPE, required = false) String sourceMimetype,
                                               @RequestParam(value = TARGET_MIMETYPE, required = false) String targetMimetype,
                                               @RequestParam Map<String, String> requestParameters,
-                                              @RequestParam (value = TEST_DELAY, required = false) Long testDelay,
+                                              @RequestParam(value = TEST_DELAY, required = false) Long testDelay,
 
                                               // The TRANSFORM_NAME_PROPERTY param allows ACS legacy transformers to specify which transform to use,
                                               // It can be removed once legacy transformers are removed from ACS.
-                                              @RequestParam (value = TRANSFORM_NAME_PROPERTY, required = false) String requestTransformName)
+                                              @RequestParam(value = TRANSFORM_NAME_PROPERTY, required = false) String requestTransformName)
     {
         if (logger.isDebugEnabled())
         {
@@ -173,10 +177,27 @@ public abstract class AbstractTransformerController implements TransformControll
                     + "targetExtension: '{}', requestParameters: {}", sourceMimetype, targetMimetype, targetExtension, requestParameters);
         }
 
-        final String targetFilename = createTargetFileName(
+        final String directUrl = requestParameters.getOrDefault(DIRECT_ACCESS_URL, "");
-                sourceMultipartFile.getOriginalFilename(), targetExtension);
+
+        File sourceFile;
+        String sourceFilename;
+        if (directUrl.isBlank())
+        {
+            if (sourceMultipartFile ==  null)
+            {
+                throw new TransformException(BAD_REQUEST.value(), "Required request part 'file' is not present");
+            }
+            sourceFile = createSourceFile(request, sourceMultipartFile);
+            sourceFilename = sourceMultipartFile.getOriginalFilename();
+        }
+        else
+        {
+            sourceFile = getSourceFileFromDirectUrl(directUrl);
+            sourceFilename = sourceFile.getName();
+        }
+
+        final String targetFilename = createTargetFileName(sourceFilename, targetExtension);
         getProbeTestTransform().incrementTransformerCount();
-        final File sourceFile = createSourceFile(request, sourceMultipartFile);
         final File targetFile = createTargetFile(request, targetFilename);
 
         Map<String, String> transformOptions = getTransformOptions(requestParameters);
@@ -191,6 +212,25 @@ public abstract class AbstractTransformerController implements TransformControll
         return body;
     }
 
+    private File getSourceFileFromDirectUrl(String directUrl)
+    {
+        File sourceFile = createTempFile("tmp", ".tmp");
+        try
+        {
+            FileUtils.copyURLToFile(new URL(directUrl), sourceFile);
+        }
+        catch (IllegalArgumentException e)
+        {
+            throw new TransformException(BAD_REQUEST.value(), "Direct Access Url is invalid.", e);
+        }
+        catch (IOException e)
+        {
+            throw new TransformException(BAD_REQUEST.value(), "Direct Access Url not found.", e);
+        }
+
+        return sourceFile;
+    }
+
     protected Map<String, String> getTransformOptions(Map<String, String> requestParameters)
     {
         Map<String, String> transformOptions = new HashMap<>(requestParameters);
@@ -245,7 +285,15 @@ public abstract class AbstractTransformerController implements TransformControll
         File sourceFile;
         try
         {
-            sourceFile = loadSourceFile(request.getSourceReference(), request.getSourceExtension());
+            final String directUrl = request.getTransformRequestOptions().getOrDefault(DIRECT_ACCESS_URL, "");
+            if (directUrl.isBlank())
+            {
+                sourceFile = loadSourceFile(request.getSourceReference(), request.getSourceExtension());
+            }
+            else
+            {
+                sourceFile = getSourceFileFromDirectUrl(directUrl);
+            }
         }
         catch (TransformException e)
         {
@@ -285,7 +333,7 @@ public abstract class AbstractTransformerController implements TransformControll
         {
             String targetMimetype = request.getTargetMediaType();
             String sourceMimetype = request.getSourceMediaType();
-            Map<String, String> transformOptions = request.getTransformRequestOptions();
+            Map<String, String> transformOptions = getTransformOptions(request.getTransformRequestOptions());
             transformerDebug.logOptions(request);
             String transformName = getTransformerName(sourceFile, sourceMimetype, targetMimetype, transformOptions);
             transformImpl(transformName, sourceMimetype, targetMimetype, transformOptions, sourceFile, targetFile);

alfresco-transformer-base/src/test/java/org/alfresco/transformer/AbstractHttpRequestTest.java

@@ -26,6 +26,7 @@
  */
 package org.alfresco.transformer;
 
+import static org.alfresco.transform.client.util.RequestParamMap.DIRECT_ACCESS_URL;
 import static org.alfresco.transform.client.util.RequestParamMap.ENDPOINT_TRANSFORM;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.springframework.http.HttpMethod.POST;
@@ -90,9 +91,12 @@ public abstract class AbstractHttpRequestTest
     @Test
     public void noFileError()
     {
-        // Transformer name is not part of the title as this is checked by another handler
+        LinkedMultiValueMap<String, Object> parameters = new LinkedMultiValueMap<>();
+        parameters.add("targetExtension", ".tmp");
+
         assertTransformError(false,
-            "Required request part 'file' is not present");
+                getTransformerName() + " - Required request part 'file' is not present",
+                parameters);
     }
 
     @Test
@@ -104,10 +108,12 @@ public abstract class AbstractHttpRequestTest
     private void assertMissingParameter(String name)
     {
         assertTransformError(true,
-            getTransformerName() + " - Request parameter '" + name + "' is missing");
+            getTransformerName() + " - Request parameter '" + name + "' is missing", null);
     }
 
-    protected void assertTransformError(boolean addFile, String errorMessage)
+    protected void assertTransformError(boolean addFile,
+                                        String errorMessage,
+                                        LinkedMultiValueMap<String, Object> additionalParams)
     {
         LinkedMultiValueMap<String, Object> parameters = new LinkedMultiValueMap<>();
         if (addFile)
@@ -115,6 +121,11 @@ public abstract class AbstractHttpRequestTest
             parameters.add("file",
                 new org.springframework.core.io.ClassPathResource("quick." + getSourceExtension()));
         }
+        if (additionalParams != null)
+        {
+            parameters.addAll(additionalParams);
+        }
+
         HttpHeaders headers = new HttpHeaders();
         headers.setContentType(MULTIPART_FORM_DATA);
         HttpEntity<LinkedMultiValueMap<String, Object>> entity = new HttpEntity<>(parameters,
@@ -123,6 +134,21 @@ public abstract class AbstractHttpRequestTest
         sendTranformationRequest(entity, errorMessage);
     }
 
+    @Test
+    public void httpTransformRequestDirectAccessUrlNotFoundTest()
+    {
+        String directUrl = "https://expired/direct/access/url";
+
+        LinkedMultiValueMap<String, Object> parameters = new LinkedMultiValueMap<>();
+        parameters.add("targetExtension", ".tmp");
+        parameters.add(DIRECT_ACCESS_URL, directUrl);
+
+        assertTransformError(false,
+                getTransformerName() + " - Direct Access Url not found.",
+                parameters);
+
+    }
+
     protected void sendTranformationRequest(
         final HttpEntity<LinkedMultiValueMap<String, Object>> entity, final String errorMessage)
     {

alfresco-transformer-base/src/test/java/org/alfresco/transformer/AbstractTransformerControllerTest.java

@@ -27,6 +27,7 @@
 package org.alfresco.transformer;
 
 import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
+import static org.alfresco.transform.client.util.RequestParamMap.DIRECT_ACCESS_URL;
 import static org.alfresco.transform.client.util.RequestParamMap.ENDPOINT_TRANSFORM;
 import static org.alfresco.transform.client.util.RequestParamMap.ENDPOINT_TRANSFORM_CONFIG_LATEST;
 import static org.alfresco.transform.client.util.RequestParamMap.ENDPOINT_TRANSFORM_CONFIG;
@@ -34,9 +35,12 @@ import static org.hamcrest.Matchers.containsString;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.when;
 import static org.springframework.http.HttpHeaders.ACCEPT;
 import static org.springframework.http.HttpHeaders.CONTENT_TYPE;
 import static org.springframework.http.HttpStatus.BAD_REQUEST;
+import static org.springframework.http.HttpStatus.CREATED;
 import static org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR;
 import static org.springframework.http.HttpStatus.OK;
 import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE;
@@ -55,6 +59,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.UUID;
 
 import org.alfresco.transform.client.model.InternalContext;
 import org.alfresco.transform.client.model.TransformReply;
@@ -68,6 +73,8 @@ import org.alfresco.transform.client.model.config.Transformer;
 import org.alfresco.transform.client.registry.TransformServiceRegistry;
 import org.alfresco.transform.router.TransformStack;
 import org.alfresco.transformer.clients.AlfrescoSharedFileStoreClient;
+import org.alfresco.transformer.model.FileRefEntity;
+import org.alfresco.transformer.model.FileRefResponse;
 import org.alfresco.transformer.probes.ProbeTestTransform;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
@@ -214,6 +221,36 @@ public abstract class AbstractTransformerControllerTest
     }
 
     protected MockHttpServletRequestBuilder mockMvcRequest(String url, MockMultipartFile sourceFile,
+                                                           String... params)
+    {
+        if (sourceFile == null)
+        {
+            return mockMvcRequestWithoutMockMultipartFile(url, params);
+        }
+        else
+        {
+            return mockMvcRequestWithMockMultipartFile(url, sourceFile, params);
+        }
+    }
+
+    private MockHttpServletRequestBuilder mockMvcRequestWithoutMockMultipartFile(String url,
+                                                           String... params)
+    {
+        MockHttpServletRequestBuilder builder = MockMvcRequestBuilders.multipart(ENDPOINT_TRANSFORM);
+
+        if (params.length % 2 != 0)
+        {
+            throw new IllegalArgumentException("each param should have a name and value.");
+        }
+        for (int i = 0; i < params.length; i += 2)
+        {
+            builder = builder.param(params[i], params[i + 1]);
+        }
+
+        return builder;
+    }
+
+    private MockHttpServletRequestBuilder mockMvcRequestWithMockMultipartFile(String url, MockMultipartFile sourceFile,
         String... params)
     {
         MockHttpServletRequestBuilder builder = MockMvcRequestBuilders.multipart(ENDPOINT_TRANSFORM).file(
@@ -569,4 +606,58 @@ public abstract class AbstractTransformerControllerTest
         transformer.setSupportedSourceAndTargetList(supportedSourceAndTargetList);
         return transformer;
     }
+
+    @Test
+    public void queueTransformRequestUsingDirectAccessUrlTest() throws Exception
+    {
+        // Files
+        String sourceFileRef = UUID.randomUUID().toString();
+        File sourceFile = getTestFile("quick." + sourceExtension, true);
+        String targetFileRef = UUID.randomUUID().toString();
+
+        TransformRequest transformRequest = createTransformRequest(sourceFileRef, sourceFile);
+        Map<String, String> transformRequestOptions = transformRequest.getTransformRequestOptions();
+
+        String directUrl = "file://" + sourceFile.toPath();
+
+        transformRequestOptions.put(DIRECT_ACCESS_URL, directUrl);
+        transformRequest.setTransformRequestOptions(transformRequestOptions);
+
+        when(alfrescoSharedFileStoreClient.saveFile(any()))
+                .thenReturn(new FileRefResponse(new FileRefEntity(targetFileRef)));
+
+        // Update the Transformation Request with any specific params before sending it
+        updateTransformRequestWithSpecificOptions(transformRequest);
+
+        // Serialize and call the transformer
+        String tr = objectMapper.writeValueAsString(transformRequest);
+        String transformationReplyAsString = mockMvc
+                .perform(MockMvcRequestBuilders
+                        .post("/transform")
+                        .header(ACCEPT, APPLICATION_JSON_VALUE)
+                        .header(CONTENT_TYPE, APPLICATION_JSON_VALUE)
+                        .content(tr))
+                .andExpect(status().is(CREATED.value()))
+                .andReturn().getResponse().getContentAsString();
+
+        TransformReply transformReply = objectMapper.readValue(transformationReplyAsString,
+                TransformReply.class);
+
+        // Assert the reply
+        assertEquals(transformRequest.getRequestId(), transformReply.getRequestId());
+        assertEquals(transformRequest.getClientData(), transformReply.getClientData());
+        assertEquals(transformRequest.getSchema(), transformReply.getSchema());
+    }
+
+    @Test
+    public void httpTransformRequestUsingDirectAccessUrlTest() throws Exception
+    {
+        File dauSourceFile = getTestFile("quick." + sourceExtension, true);
+        String directUrl = "file://" + dauSourceFile.toPath();
+
+        mockMvc.perform(
+                       mockMvcRequest("/transform", null, "targetExtension", targetExtension, DIRECT_ACCESS_URL, directUrl))
+               .andExpect(status().is(OK.value()))
+               .andExpect(content().bytes(expectedTargetFileBytes));
+    }
 }