inteligr8/alfresco-transform-core
1
0
Fork 0
mirror of https://github.com/Alfresco/alfresco-transform-core.git synced 2025-08-14 17:58:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added Fragment processing

Browse Source
This commit is contained in:
alandavis
2022-08-04 18:18:16 +01:00
parent abe1fae16a
commit 031cae74fd
16 changed files with 603 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
engines/base/src/main/java/org/alfresco/transform/base/TransformController.java
View File

@@ -296,9 +296,4 @@ public class TransformController
mav.setViewName("error"); // display error.html
return mav;
}
void resetForTesting()
{
}
}

34
engines/base/src/main/java/org/alfresco/transform/base/TransformManager.java
View File

@@ -26,7 +26,10 @@
*/
package org.alfresco.transform.base;
import org.alfresco.transform.common.TransformException;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Map;
@@ -50,28 +53,33 @@ public interface TransformManager
* The file will be deleted once the request is completed. To avoid creating extra files, if a File has already
* been created by the base t-engine, it is returned.
* If possible this method should be avoided as it is better not to leave content on disk.
* @throws IllegalStateException if this method has already been called. A call to {@link #respondWithFragment(Integer)}
* @throws IllegalStateException if this method has already been called. A call to {@link #respondWithFragment(Integer, boolean)}
* allows the method to be called again.
*/
File createTargetFile();
/**
* Allows a single transform request to have multiple transform responses. For example images from a video at
* Allows a single transform request to have multiple transform responses. For example, images from a video at
* different time offsets or different pages of a document. Following a call to this method a transform response is
* made with output that has already been generated. The {@code CustomTransformer} may then use the returned
* {@code outputStream} to generate more output for the next transform response.
* {@code CustomTransformer} to throw an {@code Exception} when it finds it has no more output.
* @param index returned with the response, so that it may be distinguished from other responses. Renditions
* use the index as an offset into elements. A {@code null} value indicates that there is no more output
* so there should not be another transform reply once the
* {@link CustomTransformer#transform(String, InputStream, String, OutputStream, Map, TransformManager)}
* returns.
* @throws IllegalStateException if a synchronous (http) request has been made as this only works with messages
* on queues.
* made with the data sent to the current {@code OutputStream}. If this method has been called, there will not be
* another response when {@link CustomTransformer#transform(String, InputStream, String, OutputStream, Map,
* TransformManager)} returns and any data written to the final {@code OutputStream} will be ignored.
* @param index returned with the response, so that the fragment may be distinguished from other responses.
* Renditions use the index as an offset into elements. A {@code null} value indicates that there
* is no more output and any data sent to the current {@code outputStream} will be ignored.
* @param finished indicates this is the final fragment. {@code False} indicates that it is expected there will be
* more fragments. There need not be a call with this parameter set to {@code true}.
* @return a new {@code OutputStream} for the next fragment. A {@code null} will be returned if {@code index} was
* {@code null} or {@code finished} was {@code true}.
* @throws TransformException if a synchronous (http) request has been made as this only works with requests
* on queues, or the first call to this method indicated there was no output, or
* another call is made after it has been indicated that there should be no more
* fragments.
* @throws IOException if there was a problem sending the response.
*/
// This works because all the state is in the TransformResponse and the t-router will just see each response as
// something to either return to the client or pass to the next stage in a pipeline. We might be able to enhance
// the logging to include the index. We may also wish to modify the client data or just make the index available
// in the message.
OutputStream respondWithFragment(Integer index);
OutputStream respondWithFragment(Integer index, boolean finished) throws IOException;
}

97
engines/base/src/main/java/org/alfresco/transform/base/transform/FragmentHandler.java Normal file
View File

@@ -0,0 +1,97 @@
/*
* #%L
* Alfresco Transform Core
* %%
* Copyright (C) 2022 - 2022 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* -
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
* -
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
* -
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
* -
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
package org.alfresco.transform.base.transform;
import org.alfresco.transform.common.TransformException;
import java.io.IOException;
import java.io.OutputStream;
import static org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR;
/**
* Separation of transform fragments logic from the {@link ProcessHandler} logic and {@link StreamHandler}.
*/
public abstract class FragmentHandler extends StreamHandler
{
private boolean methodHasBeenCall;
private boolean noMoreFragments;
protected void initTarget()
{
}
public OutputStream respondWithFragment(Integer index, boolean finished) throws IOException
{
try
{
if (index == null && !methodHasBeenCall)
{
throw new TransformException(INTERNAL_SERVER_ERROR, "No fragments were produced");
}
if (index != null && noMoreFragments)
{
throw new TransformException(INTERNAL_SERVER_ERROR, "Final fragment already sent");
}
if (index != null)
{
super.handleSuccessfulTransform();
logFragment(index, transformManager.getOutputLength());
}
}
finally
{
methodHasBeenCall = true;
noMoreFragments = noMoreFragments || index == null || finished;
}
return noMoreFragments ? null : switchToNewOutputStreamForNewFragment();
}
protected void logFragment(Integer index, Long outputLength)
{
}
@Override
protected void handleSuccessfulTransform() throws IOException
{
if (!methodHasBeenCall)
{
super.handleSuccessfulTransform();
}
}
private OutputStream switchToNewOutputStreamForNewFragment() throws IOException
{
transformManager.getOutputStream().close();
transformManager.deleteTargetFile();
initTarget();
setOutputStream();
return outputStream;
}
}

19
engines/base/src/main/java/org/alfresco/transform/base/transform/TransformProcess.java → engines/base/src/main/java/org/alfresco/transform/base/transform/ProcessHandler.java
View File

@@ -40,6 +40,7 @@ import org.springframework.web.multipart.MultipartFile;
import javax.jms.Destination;
import javax.servlet.http.HttpServletRequest;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
@@ -57,13 +58,13 @@ import static org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR;
import static org.springframework.http.HttpStatus.OK;
/**
* Provides the transform logic common to http upload/download, message and probe requests. See
* Provides the transform logic common to http (upload/download), message and probe requests. See
* {@link TransformHandler#handleHttpRequest(HttpServletRequest, MultipartFile, String, String, Map)},
* {@link TransformHandler#handleMessageRequest(TransformRequest, Long, Destination)} and
* {@link TransformHandler#handleProbRequest(String, String, Map, File, File)}. Note the handing of transform requests
* via a message queue is the same as via the {@link TransformController#transform(TransformRequest, Long)}.
*/
abstract class TransformProcess extends TransformStreamHandler
abstract class ProcessHandler extends FragmentHandler
{
private static final List<String> NON_TRANSFORM_OPTION_REQUEST_PARAMETERS = Arrays.asList(SOURCE_EXTENSION,
TARGET_EXTENSION, TARGET_MIMETYPE, SOURCE_MIMETYPE, DIRECT_ACCESS_URL);
@@ -77,7 +78,7 @@ abstract class TransformProcess extends TransformStreamHandler
private final ProbeTransform probeTransform;
private final Map<String, CustomTransformer> customTransformersByName;
TransformProcess(String sourceMimetype, String targetMimetype, Map<String, String> transformOptions,
ProcessHandler(String sourceMimetype, String targetMimetype, Map<String, String> transformOptions,
String reference, TransformServiceRegistry transformRegistry, TransformerDebug transformerDebug,
ProbeTransform probeTransform, Map<String, CustomTransformer> customTransformersByName)
{
@@ -100,6 +101,13 @@ abstract class TransformProcess extends TransformStreamHandler
return transformOptions;
}
@Override
protected void init() throws IOException
{
transformManager.setProcessHandler(this);
super.init();
}
public void handleTransformRequest()
{
LogEntry.start();
@@ -137,6 +145,11 @@ abstract class TransformProcess extends TransformStreamHandler
}
}
protected void logFragment(Integer index, Long outputLength)
{
transformerDebug.logFragment(reference, index, outputLength);
}
@Override
public void transform(CustomTransformer customTransformer) throws Exception
{

25
engines/base/src/main/java/org/alfresco/transform/base/transform/TransformStreamHandler.java → engines/base/src/main/java/org/alfresco/transform/base/transform/StreamHandler.java
View File

@@ -34,16 +34,16 @@ import java.io.InputStream;
import java.io.OutputStream;
/**
* Separation of InputStream, OutputStream, sourceFile and targetFile from the {@link TransformProcess} logic. Allows
* Separation of InputStream, OutputStream, sourceFile and targetFile from the {@link ProcessHandler} logic. Allows
* {@link CustomTransformer} implementations to call {@link TransformManager#createSourceFile()} and
* {@link TransformManager#createTargetFile()} so that extra Files are not created if there was one already in
* existence.
*
* Subclasses MUST call transformManager.setSourceFile(File) and transformManager.setSourceFile(File) if they start
* with files rather than streams, before calling the {@link #init()} method which calls
* transformManager.setInputStream(InputStream) and transformManager.setOutputStream(OutputStream).
* transformManager.setOutputStream(InputStream) and transformManager.setOutputStream(OutputStream).
*/
public abstract class TransformStreamHandler
public abstract class StreamHandler
{
protected TransformManagerImpl transformManager = new TransformManagerImpl();
protected InputStream inputStream;
@@ -52,8 +52,18 @@ public abstract class TransformStreamHandler
public abstract void handleTransformRequest() throws Exception;
protected void init() throws IOException
{
setInputStream();
setOutputStream();
}
private void setInputStream() throws IOException
{
inputStream = transformManager.setInputStream(getInputStream());
}
protected void setOutputStream() throws IOException
{
outputStream = transformManager.setOutputStream(getOutputStream());
}
@@ -66,8 +76,7 @@ public abstract class TransformStreamHandler
try
{
transform(customTransformer);
transformManager.copyTargetFileToOutputStream();
onSuccessfulTransform();
handleSuccessfulTransform();
}
finally
{
@@ -79,6 +88,12 @@ public abstract class TransformStreamHandler
protected abstract void transform(CustomTransformer customTransformer) throws Exception;
protected void handleSuccessfulTransform() throws IOException
{
transformManager.copyTargetFileToOutputStream();
onSuccessfulTransform();
}
protected void onSuccessfulTransform()
{
}

25
engines/base/src/main/java/org/alfresco/transform/base/transform/TransformHandler.java
View File

@@ -165,7 +165,7 @@ public class TransformHandler
{
AtomicReference<ResponseEntity<Resource>> responseEntity = new AtomicReference<>();
new TransformProcess(sourceMimetype, targetMimetype, requestParameters,
new ProcessHandler(sourceMimetype, targetMimetype, requestParameters,
"e" + httpRequestCount.getAndIncrement(), transformRegistry,
transformerDebug, getProbeTransform(), customTransformersByName)
{
@@ -209,7 +209,7 @@ public class TransformHandler
public void handleProbRequest(String sourceMimetype, String targetMimetype, Map<String, String> transformOptions,
File sourceFile, File targetFile)
{
new TransformProcess(sourceMimetype, targetMimetype, transformOptions,
new ProcessHandler(sourceMimetype, targetMimetype, transformOptions,
"p" + httpRequestCount.getAndIncrement(), transformRegistry,
transformerDebug, getProbeTransform(), customTransformersByName)
{
@@ -245,7 +245,7 @@ public class TransformHandler
public TransformReply handleMessageRequest(TransformRequest request, Long timeout, Destination replyToQueue)
{
TransformReply reply = createBasicTransformReply(request);
new TransformProcess(request.getSourceMediaType(), request.getTargetMediaType(),
new ProcessHandler(request.getSourceMediaType(), request.getTargetMediaType(),
request.getTransformRequestOptions(),"unset", transformRegistry,
transformerDebug, getProbeTransform(), customTransformersByName)
{
@@ -254,10 +254,16 @@ public class TransformHandler
{
checkTransformRequestValid(request, reply);
reference = TransformStack.getReference(reply.getInternalContext());
transformManager.setTargetFile(createTargetFile(null, sourceMimetype, targetMimetype));
initTarget();
super.init();
}
@Override
protected void initTarget()
{
transformManager.setTargetFile(createTargetFile(null, sourceMimetype, targetMimetype));
}
@Override
protected InputStream getInputStream()
{
@@ -425,7 +431,7 @@ public class TransformHandler
}
catch (HttpClientErrorException e)
{
throw new TransformException(e.getStatusCode(), messageWithCause("Failed to read the source", e));
throw new TransformException(e.getStatusCode(), messageWithCause("Failed to read the source from the SFS", e));
}
}
@@ -461,9 +467,12 @@ public class TransformHandler
private static String messageWithCause(final String prefix, Throwable e)
{
final StringBuilder sb = new StringBuilder();
sb.append(prefix).append(" - ")
.append(e.getClass().getSimpleName()).append(": ")
.append(e.getMessage());
sb.append(prefix).append(" - ");
if (e.getClass() != TransformException.class)
{
sb.append(e.getClass().getSimpleName()).append(": ");
}
sb.append(e.getMessage());
while (e.getCause() != null)
{

19
engines/base/src/main/java/org/alfresco/transform/base/transform/TransformManagerImpl.java
View File

@@ -48,6 +48,7 @@ public class TransformManagerImpl implements TransformManager
private static final Logger logger = LoggerFactory.getLogger(TransformManagerImpl.class);
private HttpServletRequest request;
private ProcessHandler processHandler;
private InputStream inputStream;
private OutputStreamLengthRecorder outputStreamLengthRecorder;
private String sourceMimetype;
@@ -65,6 +66,11 @@ public class TransformManagerImpl implements TransformManager
this.request = request;
}
public void setProcessHandler(ProcessHandler processHandler)
{
this.processHandler = processHandler;
}
public InputStream setInputStream(InputStream inputStream)
{
this.inputStream = inputStream;
@@ -194,7 +200,10 @@ public class TransformManagerImpl implements TransformManager
{
logger.error("Failed to delete temporary source file "+sourceFile.getPath());
}
outputStreamLengthRecorder = null;
sourceFile = null;
createSourceFileCalled = false;
startedWithSourceFile = null;
}
public void deleteTargetFile()
@@ -204,18 +213,18 @@ public class TransformManagerImpl implements TransformManager
logger.error("Failed to delete temporary target file "+targetFile.getPath());
}
targetFile = null;
createTargetFileCalled = false;
startedWithTargetFile = null;
}
@Override
public OutputStream respondWithFragment(Integer index)
public OutputStream respondWithFragment(Integer index, boolean finished) throws IOException
{
if (request != null)
{
throw new IllegalStateException(
" Fragments may only be sent with asynchronous requests. This a synchronous http request");
throw new IllegalStateException("Fragments may only be sent via message queues. This an http request");
}
// TODO send the current output as a TransformResponse and then start a new one.
throw new UnsupportedOperationException("Not currently supported");
return processHandler.respondWithFragment(index, finished);
}
}

20
engines/base/src/test/java/org/alfresco/transform/base/TransformControllerTest.java
View File

@@ -40,7 +40,7 @@ import org.alfresco.transform.base.fakes.FakeTransformerTxT2Pdf;
import org.alfresco.transform.base.model.FileRefEntity;
import org.alfresco.transform.base.model.FileRefResponse;
import org.alfresco.transform.base.transform.TransformHandler;
import org.alfresco.transform.base.transform.TransformHandlerTest;
import org.alfresco.transform.base.transform.FragmentTest;
import org.alfresco.transform.client.model.TransformReply;
import org.alfresco.transform.client.model.TransformRequest;
import org.alfresco.transform.config.TransformConfig;
@@ -105,8 +105,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
/**
* Tests the endpoints of the TransformController. Testing of transforms is limited as in that is covered by
* {@link TransformHandlerTest}.
* Tests the endpoints of the TransformController.
*
* Also see {@link TransformControllerAllInOneTest}.
*/
@@ -131,8 +130,7 @@ public class TransformControllerTest
@MockBean
protected AlfrescoSharedFileStoreClient fakeSfsClient;
@BeforeEach
public void fakeSfsClient()
private void fakeSfsClient()
{
final Map<String,File> sfsRef2File = new HashMap<>();
when(fakeSfsClient.saveFile(any())).thenAnswer((Answer) invocation -> {
@@ -151,7 +149,6 @@ public class TransformControllerTest
.body((Resource) new UrlResource(sfsRef2File.get(invocation.getArguments()[0]).toURI())));
}
static void resetProbeForTesting(TransformController transformController)
{
transformController.transformHandler.getProbeTransform().resetForTesting();
@@ -305,6 +302,7 @@ public class TransformControllerTest
@Test
public void testTransformEndpointThatUsesTransformRequests() throws Exception
{
fakeSfsClient();
File sourceFile = getTestFile("original.txt", true, tempDir);
String sourceFileRef = fakeSfsClient.saveFile(sourceFile).getEntry().getFileRef();
@@ -358,11 +356,11 @@ public class TransformControllerTest
@Test
public void testTestTransformEndpointWhichConvertsRequestParameters() throws Exception
{
TransformHandler orig = transformController.transformHandler;
TransformHandler transformHandlerOrig = transformController.transformHandler;
try
{
TransformHandler spy = spy(orig);
transformController.transformHandler = spy;
TransformHandler transformHandlerSpy = spy(transformHandlerOrig);
transformController.transformHandler = transformHandlerSpy;
mockMvc.perform(
MockMvcRequestBuilders.multipart(ENDPOINT_TEST)
@@ -377,7 +375,7 @@ public class TransformControllerTest
.param("name2", PAGE_REQUEST_PARAM).param("value2", "1")
.param("name3", SOURCE_ENCODING).param("value3", "UTF-8"));
verify(spy).handleHttpRequest(any(), any(), eq(MIMETYPE_TEXT_PLAIN), eq(MIMETYPE_PDF),
verify(transformHandlerSpy).handleHttpRequest(any(), any(), eq(MIMETYPE_TEXT_PLAIN), eq(MIMETYPE_PDF),
eq(ImmutableMap.of(
SOURCE_MIMETYPE, MIMETYPE_TEXT_PLAIN,
TARGET_MIMETYPE, MIMETYPE_PDF,
@@ -386,7 +384,7 @@ public class TransformControllerTest
}
finally
{
transformController.transformHandler = orig;
transformController.transformHandler = transformHandlerOrig;
}
}

4
engines/base/src/test/java/org/alfresco/transform/base/fakes/AbstractFakeTransformer.java
View File

@@ -39,7 +39,7 @@ import java.util.Map;
import java.util.stream.Collectors;
/**
* Subclass MUST be named TestTransformer\<something>. Appends the name of the CustomTransformer and any t-options
* Subclass MUST be named FakeTransformer\<something>. Appends the name of the CustomTransformer and any t-options
* to the output. The output is always a String regardless of the stated mimetypes.
*/
@TestComponent
@@ -51,7 +51,7 @@ public abstract class AbstractFakeTransformer implements CustomTransformer
public String getTransformerName()
{
String simpleClassName = getClass().getSimpleName();
return simpleClassName.substring("TestTransformer".length());
return simpleClassName.substring("FakeTransformer".length());
}
@Override

39
engines/base/src/test/java/org/alfresco/transform/base/fakes/FakeTransformEngineWithFragments.java Normal file
View File

@@ -0,0 +1,39 @@
package org.alfresco.transform.base.fakes;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import org.alfresco.transform.base.probes.ProbeTransform;
import org.alfresco.transform.config.SupportedSourceAndTarget;
import org.alfresco.transform.config.TransformConfig;
import org.alfresco.transform.config.Transformer;
import java.util.Collections;
import static org.alfresco.transform.common.Mimetype.MIMETYPE_IMAGE_JPEG;
import static org.alfresco.transform.common.Mimetype.MIMETYPE_PDF;
import static org.alfresco.transform.common.Mimetype.MIMETYPE_TEXT_PLAIN;
public class FakeTransformEngineWithFragments extends AbstractFakeTransformEngine
{
@Override public TransformConfig getTransformConfig()
{
String imageOptions = "imageOptions";
return TransformConfig.builder()
.withTransformers(ImmutableList.of(
Transformer.builder()
.withTransformerName("Fragments")
.withSupportedSourceAndTargetList(ImmutableSet.of(
SupportedSourceAndTarget.builder()
.withSourceMediaType(MIMETYPE_PDF)
.withTargetMediaType(MIMETYPE_IMAGE_JPEG)
.build()))
.build()))
.build();
}
@Override public ProbeTransform getProbeTransform()
{
return new ProbeTransform("probe.pdf", MIMETYPE_PDF, MIMETYPE_IMAGE_JPEG, Collections.emptyMap(),
60, 16, 400, 10240, 60 * 30 + 1, 60 * 15 + 20);
}
}

87
engines/base/src/test/java/org/alfresco/transform/base/fakes/FakeTransformerFragments.java Normal file
View File

@@ -0,0 +1,87 @@
/*
* #%L
* Alfresco Transform Core
* %%
* Copyright (C) 2005 - 2022 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* -
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
* -
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
* -
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
* -
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
package org.alfresco.transform.base.fakes;
import org.alfresco.transform.base.TransformManager;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.util.Map;
/**
* Returns lines in the supplied input as a sequence of transform result fragments.
* - If the current line is {@code "Null"} no output is made and {@code null} is passed as the {@code index} to
* {@link TransformManager#respondWithFragment(Integer, boolean)}. The {code finished} parameter is unset.
* - If {@code "Finished"}, the text is written and the {code finished} parameter is set.
* - If the current line is {@code "NullFinished"} no output is made and {@code null} is passed as the {@code index} to
* {@code respondWithFragment}. The {code finished} parameter is set.
* - If {@code "Ignored"} it will be written to the output, but the {@code respondWithFragment} method will not be
* called, so should be ignored if the final line.
* If the input is "WithoutFragments", {@code respondWithFragment} is not called.
*/
public class FakeTransformerFragments extends AbstractFakeTransformer
{
@Override
public void transform(String sourceMimetype, InputStream inputStream, String targetMimetype,
OutputStream outputStream, Map<String, String> transformOptions, TransformManager transformManager)
throws Exception
{
String input = new String(inputStream.readAllBytes(), StandardCharsets.UTF_8);
String[] lines = input.split("\n");
if ("WithoutFragments".equals(input))
{
write(outputStream, input);
}
else
{
for (int i = 0; i < lines.length; i++)
{
String line = lines[i];
Integer index = "Null".equals(line) || "NullFinished".equals(line) ? null : i;
boolean finished = "Finished".equals(line) || "NullFinished".equals(line);
if (index != null)
{
write(outputStream, line);
}
if (!"Ignored".equals(line)) {
outputStream = transformManager.respondWithFragment(index, finished);
}
}
}
}
private void write(OutputStream outputStream, String text) throws IOException
{
if (outputStream != null)
{
byte[] bytes = text.getBytes(StandardCharsets.UTF_8);
outputStream.write(bytes, 0, bytes.length);
}
}
}

1
engines/base/src/test/java/org/alfresco/transform/base/http/RestTest.java
View File

@@ -29,7 +29,6 @@ package org.alfresco.transform.base.http;
import org.alfresco.transform.base.fakes.FakeTransformEngineWithTwoCustomTransformers;
import org.alfresco.transform.base.fakes.FakeTransformerPdf2Png;
import org.alfresco.transform.base.fakes.FakeTransformerTxT2Pdf;
import org.alfresco.transform.base.transform.TransformHandlerTest;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;

237
engines/base/src/test/java/org/alfresco/transform/base/transform/FragmentTest.java Normal file
View File

@@ -0,0 +1,237 @@
/*
* #%L
* Alfresco Transform Core
* %%
* Copyright (C) 2022 - 2022 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* -
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
* -
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
* -
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
* -
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
package org.alfresco.transform.base.transform;
import com.google.common.collect.ImmutableList;
import org.alfresco.transform.base.clients.AlfrescoSharedFileStoreClient;
import org.alfresco.transform.base.fakes.FakeTransformEngineWithFragments;
import org.alfresco.transform.base.fakes.FakeTransformerFragments;
import org.alfresco.transform.base.messaging.TransformReplySender;
import org.alfresco.transform.base.model.FileRefEntity;
import org.alfresco.transform.base.model.FileRefResponse;
import org.alfresco.transform.client.model.TransformReply;
import org.alfresco.transform.client.model.TransformRequest;
import org.apache.commons.lang3.tuple.Pair;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.core.io.ByteArrayResource;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.mock.web.MockMultipartFile;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.web.servlet.MockMvc;
import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
import javax.jms.Destination;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import static org.alfresco.transform.base.transform.StreamHandlerTest.read;
import static org.alfresco.transform.common.Mimetype.MIMETYPE_IMAGE_JPEG;
import static org.alfresco.transform.common.Mimetype.MIMETYPE_PDF;
import static org.alfresco.transform.common.Mimetype.MIMETYPE_TEXT_PLAIN;
import static org.alfresco.transform.common.RequestParamMap.ENDPOINT_TRANSFORM;
import static org.alfresco.transform.common.RequestParamMap.SOURCE_MIMETYPE;
import static org.alfresco.transform.common.RequestParamMap.TARGET_MIMETYPE;
import static org.hamcrest.Matchers.containsString;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.springframework.http.HttpStatus.OK;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
@AutoConfigureMockMvc
@SpringBootTest(classes={org.alfresco.transform.base.Application.class})
@ContextConfiguration(classes = {
FakeTransformEngineWithFragments.class,
FakeTransformerFragments.class})
public class FragmentTest
{
@Autowired
private TransformHandler transformHandler;
@Autowired
private MockMvc mockMvc;
@MockBean
protected AlfrescoSharedFileStoreClient fakeSfsClient;
@MockBean
private TransformReplySender transformReplySender;
private void assertFragments(String sourceText, String expectedError, List<String> expectedLines)
{
List<Pair<Destination, TransformReply>> replies = new ArrayList<>();
List<String> lines = new ArrayList<>();
String sourceReference = UUID.randomUUID().toString();
String targetReference = UUID.randomUUID().toString();
when(fakeSfsClient.retrieveFile(any()))
.thenReturn(new ResponseEntity<>(new ByteArrayResource(sourceText.getBytes(StandardCharsets.UTF_8)),
new HttpHeaders(), OK));
when(fakeSfsClient.saveFile(any()))
.thenAnswer(invocation ->
{
lines.add(read(invocation.getArgument(0)));
return new FileRefResponse(new FileRefEntity(targetReference));
});
doAnswer(invocation ->
{
replies.add(Pair.of(invocation.getArgument(0), invocation.getArgument(1)));
return null;
}).when(transformReplySender).send(any(), any());
TransformRequest request = TransformRequest
.builder()
.withRequestId(UUID.randomUUID().toString())
.withSourceMediaType(MIMETYPE_PDF)
.withTargetMediaType(MIMETYPE_IMAGE_JPEG)
.withTargetExtension("jpeg")
.withSchema(1)
.withClientData("ACS")
.withSourceReference(sourceReference)
.withSourceSize(32L)
.withInternalContextForTransformEngineTests()
.build();
transformHandler.handleMessageRequest(request, Long.MAX_VALUE, null);
TransformReply lastReply = replies.get(replies.size() - 1).getRight();
String errorDetails = lastReply.getErrorDetails();
int status = lastReply.getStatus();
if (expectedError == null)
{
assertNull(errorDetails);
assertEquals(HttpStatus.CREATED.value(), status);
}
else
{
assertEquals("Transform failed - "+expectedError, errorDetails);
assertEquals(HttpStatus.INTERNAL_SERVER_ERROR.value(), status);
}
assertEquals(expectedLines, lines);
}
@Test
public void testErrorIfHttp() throws Exception
{
String expectedError = "Fragments may only be sent via message queues. This an http request";
mockMvc.perform(
MockMvcRequestBuilders.multipart(ENDPOINT_TRANSFORM)
.file(new MockMultipartFile("file", null, MIMETYPE_TEXT_PLAIN,
"Start".getBytes(StandardCharsets.UTF_8)))
.param(SOURCE_MIMETYPE, MIMETYPE_PDF)
.param(TARGET_MIMETYPE, MIMETYPE_IMAGE_JPEG))
.andExpect(status().isInternalServerError())
.andExpect(status().reason(containsString(expectedError)));
}
@Test
public void testWithoutCallingRespondWithFragment()
{
assertFragments("WithoutFragments", null, ImmutableList.of("WithoutFragments"));
}
@Test
public void testSingleRespondWithFragmentCall()
{
assertFragments("Finished", null, ImmutableList.of("Finished"));
}
@Test
public void testMultipleFragmentCallsWithFinished()
{
assertFragments("line1\nline2\nFinished", null,
ImmutableList.of("line1", "line2", "Finished"));
}
@Test
public void testMultipleFragmentsCallsWithoutFinish()
{
assertFragments("line1\nline2\nline3", null,
ImmutableList.of("line1", "line2", "line3"));
}
@Test
public void testMultipleFragmentsCallsWithoutSendingLastFragment()
{
assertFragments("line1\nline2\nline3\nIgnored", null,
ImmutableList.of("line1", "line2", "line3"));
}
@Test
public void testNoFragments()
{
assertFragments("NullFinished", "No fragments were produced", ImmutableList.of());
}
@Test
public void testEndTooEarlyUsingFinished()
{
assertFragments("line1\nFinished\nline3", "Final fragment already sent",
ImmutableList.of("line1", "Finished"));
}
@Test
public void testEndTooEarlyUsingNull()
{
assertFragments("line1\nNull\nline3", "Final fragment already sent",
ImmutableList.of("line1"));
}
@Test
public void testFinishedAndNull()
{
// Able to just ignore the extra null call that request nothing
assertFragments("line1\nFinished\nNull", null, ImmutableList.of("line1", "Finished"));
}
@Test
public void testNullAndNull()
{
// Able to just ignore the extra null call that request nothing
assertFragments("line1\nNull\nNull", null, ImmutableList.of("line1"));
}
@Test
public void testNullAndFinished()
{
assertFragments("line1\nNull\nFinished", "Final fragment already sent",
ImmutableList.of("line1"));
}
}

56
engines/base/src/test/java/org/alfresco/transform/base/transform/TransformStreamHandlerTest.java → engines/base/src/test/java/org/alfresco/transform/base/transform/StreamHandlerTest.java
View File

@@ -48,10 +48,10 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* Tests {@link TransformStreamHandler}, {@link TransformManagerImpl#createSourceFile()} and
* Tests {@link StreamHandler}, {@link TransformManagerImpl#createSourceFile()} and
* {@link TransformManagerImpl#createTargetFile()} methods.
*/
public class TransformStreamHandlerTest
public class StreamHandlerTest
{
public static final String ORIGINAL = "Original";
public static final String CHANGE = " plus some change";
@@ -81,7 +81,7 @@ public class TransformStreamHandlerTest
return File.createTempFile("temp_", null, tempDir);
}
private void write(File file, String text) throws IOException
private static void write(File file, String text) throws IOException
{
try (OutputStream outputStream = new BufferedOutputStream(new FileOutputStream(file)))
{
@@ -89,13 +89,13 @@ public class TransformStreamHandlerTest
}
}
private void write(OutputStream outputStream, String text) throws IOException
private static void write(OutputStream outputStream, String text) throws IOException
{
byte[] bytes = text.getBytes(StandardCharsets.ISO_8859_1);
outputStream.write(bytes, 0, bytes.length);
}
private String read(File file) throws IOException
public static String read(File file) throws IOException
{
try (InputStream inputStream = new BufferedInputStream(new FileInputStream(file)))
{
@@ -103,12 +103,12 @@ public class TransformStreamHandlerTest
}
}
private String read(InputStream inputStream) throws IOException
private static String read(InputStream inputStream) throws IOException
{
return new String(inputStream.readAllBytes(), StandardCharsets.ISO_8859_1);
}
private String read(ByteArrayOutputStream outputStream)
private static String read(ByteArrayOutputStream outputStream)
{
return outputStream.toString(StandardCharsets.ISO_8859_1);
}
@@ -142,11 +142,12 @@ public class TransformStreamHandlerTest
transformManager.copyTargetFileToOutputStream();
transformManager.getOutputStream().close();
closeInputStreamWithoutException(inputStream);
Long outputLength = transformManager.getOutputLength();
transformManager.deleteSourceFile();
transformManager.deleteTargetFile();
assertEquals(EXPECTED, read(outputStream));
assertEquals(EXPECTED.length(), transformManager.getOutputLength());
assertEquals(EXPECTED.length(), outputLength);
}
}
@@ -166,11 +167,12 @@ public class TransformStreamHandlerTest
transformManager.copyTargetFileToOutputStream();
transformManager.getOutputStream().close();
closeInputStreamWithoutException(inputStream);
Long outputLength = transformManager.getOutputLength();
transformManager.deleteSourceFile();
transformManager.deleteTargetFile();
assertEquals(EXPECTED, read(outputStream));
assertEquals(EXPECTED.length(), transformManager.getOutputLength());
assertEquals(EXPECTED.length(), outputLength);
assertFalse(sourceFileCreatedByTransform.exists());
}
}
@@ -193,11 +195,12 @@ public class TransformStreamHandlerTest
transformManager.copyTargetFileToOutputStream();
closeInputStreamWithoutException(inputStream);
transformManager.getOutputStream().close();
Long outputLength = transformManager.getOutputLength();
transformManager.deleteSourceFile();
transformManager.deleteTargetFile();
assertEquals(EXPECTED, read(outputStream));
assertEquals(EXPECTED.length(), transformManager.getOutputLength());
assertEquals(EXPECTED.length(), outputLength);
assertFalse(sourceFile.exists());
}
}
@@ -222,11 +225,12 @@ public class TransformStreamHandlerTest
transformManager.copyTargetFileToOutputStream();
closeInputStreamWithoutException(inputStream);
transformManager.getOutputStream().close();
Long outputLength = transformManager.getOutputLength();
transformManager.deleteSourceFile();
transformManager.deleteTargetFile();
assertEquals(EXPECTED, read(outputStream));
assertEquals(EXPECTED.length(), transformManager.getOutputLength());
assertEquals(EXPECTED.length(), outputLength);
assertFalse(sourceFile.exists());
}
}
@@ -253,11 +257,12 @@ public class TransformStreamHandlerTest
transformManager.copyTargetFileToOutputStream();
transformManager.getOutputStream().close();
closeInputStreamWithoutException(inputStream);
Long outputLength = transformManager.getOutputLength();
transformManager.deleteSourceFile();
transformManager.deleteTargetFile();
assertEquals(EXPECTED, read(outputStream));
assertEquals(EXPECTED.length(), transformManager.getOutputLength());
assertEquals(EXPECTED.length(), outputLength);
assertFalse(targetFileCreatedByTransform.exists());
}
}
@@ -280,11 +285,12 @@ public class TransformStreamHandlerTest
transformManager.getOutputStream().close();
closeInputStreamWithoutException(inputStream);
String actual = read(targetFile);
Long outputLength = transformManager.getOutputLength();
transformManager.deleteSourceFile();
transformManager.deleteTargetFile();
assertEquals(EXPECTED, actual);
assertEquals(EXPECTED.length(), transformManager.getOutputLength());
assertEquals(EXPECTED.length(), outputLength);
assertFalse(targetFile.exists());
}
}
@@ -309,11 +315,12 @@ public class TransformStreamHandlerTest
transformManager.getOutputStream().close();
closeInputStreamWithoutException(inputStream);
String actual = read(targetFile);
Long outputLength = transformManager.getOutputLength();
transformManager.deleteSourceFile();
transformManager.deleteTargetFile();
assertEquals(EXPECTED, actual);
assertEquals(EXPECTED.length(), transformManager.getOutputLength());
assertEquals(EXPECTED.length(), outputLength);
assertFalse(targetFile.exists());
}
}
@@ -344,11 +351,12 @@ public class TransformStreamHandlerTest
transformManager.copyTargetFileToOutputStream();
closeInputStreamWithoutException(inputStream);
transformManager.getOutputStream().close();
Long outputLength = transformManager.getOutputLength();
transformManager.deleteSourceFile();
transformManager.deleteTargetFile();
assertEquals(EXPECTED, read(targetFile));
assertEquals(EXPECTED.length(), transformManager.getOutputLength());
assertEquals(EXPECTED.length(), outputLength);
assertFalse(sourceFile.exists());
assertTrue(targetFile.exists());
}
@@ -375,11 +383,12 @@ public class TransformStreamHandlerTest
closeInputStreamWithoutException(inputStream);
transformManager.getOutputStream().close();
String actual = read(targetFile);
Long outputLength = transformManager.getOutputLength();
transformManager.deleteSourceFile();
transformManager.deleteTargetFile();
assertEquals(EXPECTED, actual);
assertEquals(EXPECTED.length(), transformManager.getOutputLength());
assertEquals(EXPECTED.length(), outputLength);
assertFalse(sourceFile.exists());
assertFalse(targetFile.exists());
}
@@ -403,16 +412,17 @@ public class TransformStreamHandlerTest
closeInputStreamWithoutException(inputStream);
transformManager.getOutputStream().close();
String actual = read(targetFile);
Long outputLength = transformManager.getOutputLength();
transformManager.deleteSourceFile();
transformManager.deleteTargetFile();
assertEquals(EXPECTED, actual);
assertEquals(EXPECTED.length(), transformManager.getOutputLength());
assertEquals(EXPECTED.length(), outputLength);
assertFalse(targetFile.exists());
}
}
private abstract class FakeTransformStreamHandler extends TransformStreamHandler
private abstract class FakeStreamHandler extends StreamHandler
{
@Override
public void handleTransformRequest() throws Exception
@@ -435,7 +445,7 @@ public class TransformStreamHandlerTest
try (ByteArrayOutputStream os = new ByteArrayOutputStream())
{
new FakeTransformStreamHandler()
new FakeStreamHandler()
{
@Override
protected void init() throws IOException
@@ -467,7 +477,7 @@ public class TransformStreamHandlerTest
File sourceFile = tempFile();
write(sourceFile, ORIGINAL);
new FakeTransformStreamHandler()
new FakeStreamHandler()
{
@Override
protected void init() throws IOException
@@ -499,7 +509,7 @@ public class TransformStreamHandlerTest
File sourceFile = tempFile();
write(sourceFile, ORIGINAL);
new FakeTransformStreamHandler()
new FakeStreamHandler()
{
@Override
protected InputStream getInputStream() throws IOException
@@ -520,7 +530,7 @@ public class TransformStreamHandlerTest
{
File targetFile = tempFile();
new FakeTransformStreamHandler()
new FakeStreamHandler()
{
@Override
protected InputStream getInputStream()
@@ -547,7 +557,7 @@ public class TransformStreamHandlerTest
{
try (ByteArrayOutputStream os = new ByteArrayOutputStream())
{
new FakeTransformStreamHandler()
new FakeStreamHandler()
{
@Override
protected InputStream getInputStream()

73
engines/base/src/test/java/org/alfresco/transform/base/transform/TransformHandlerTest.java
View File

@@ -26,82 +26,15 @@
*/
package org.alfresco.transform.base.transform;
import org.alfresco.transform.base.transform.TransformHandler;
import org.alfresco.transform.base.transform.TransformProcess;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.web.servlet.MockMvc;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import org.alfresco.transform.base.TransformControllerTest;
/**
* Tests {@link TransformHandler} and {@link TransformProcess}.
* Tests {@link TransformHandler}, {@link ProcessHandler} and {@link TransformManagerImpl}.
* Note {@link TransformControllerTest} already provides 'good path' coverage.
*/
public class TransformHandlerTest
{
@Autowired
private MockMvc mockMvc;
@Autowired
private TransformHandler transformHandler;
// @Test
// public void transformRequestUsingDirectAccessUrlTest() throws Exception
// {
// // Files
// String sourceFileRef = UUID.randomUUID().toString();
// File sourceFile = getTestFile("quick." + sourceExtension, true);
// String targetFileRef = UUID.randomUUID().toString();
//
// TransformRequest transformRequest = createTransformRequest(sourceFileRef, sourceFile);
// Map<String, String> transformRequestOptions = transformRequest.getTransformRequestOptions();
//
// String directUrl = "file://" + sourceFile.toPath();
//
// transformRequestOptions.put(DIRECT_ACCESS_URL, directUrl);
// transformRequest.setTransformRequestOptions(transformRequestOptions);
//
// when(alfrescoSharedFileStoreClient.saveFile(any()))
// .thenReturn(new FileRefResponse(new FileRefEntity(targetFileRef)));
//
// // Update the Transformation Request with any specific params before sending it
// updateTransformRequestWithSpecificOptions(transformRequest);
//
// // Serialize and call the transformer
// String tr = objectMapper.writeValueAsString(transformRequest);
// String transformationReplyAsString = mockMvc
// .perform(MockMvcRequestBuilders
// .post("/transform")
// .header(ACCEPT, APPLICATION_JSON_VALUE)
// .header(CONTENT_TYPE, APPLICATION_JSON_VALUE)
// .content(tr))
// .andExpect(status().is(CREATED.value()))
// .andReturn().getResponse().getContentAsString();
//
// TransformReply transformReply = objectMapper.readValue(transformationReplyAsString,
// TransformReply.class);
//
// // Assert the reply
// assertEquals(transformRequest.getRequestId(), transformReply.getRequestId());
// assertEquals(transformRequest.getClientData(), transformReply.getClientData());
// assertEquals(transformRequest.getSchema(), transformReply.getSchema());
// }
//
// @Test
// public void httpTransformRequestUsingDirectAccessUrlTest() throws Exception
// {
// File dauSourceFile = getTestFile("quick." + sourceExtension, true);
// String directUrl = "file://" + dauSourceFile.toPath();
//
// ResultActions resultActions = mockMvc.perform(
// mockMvcRequest(ENDPOINT_TRANSFORM, null)
// .param("targetExtension", targetExtension)
// .param(DIRECT_ACCESS_URL, directUrl))
// .andExpect(status().is(OK.value()));
//
// if (expectedTargetFileBytes != null)
// {
// resultActions.andExpect(content().bytes(expectedTargetFileBytes));
// }
// }
}

8
model/src/main/java/org/alfresco/transform/common/TransformerDebug.java
View File

@@ -184,6 +184,14 @@ public class TransformerDebug
" " + key + "=\"" + value.replaceAll("\"", "\\\"") + "\"";
}
public void logFragment(String reference, int index, long size)
{
if (logger.isDebugEnabled())
{
logger.debug(getPaddedReference(reference) + " fragment["+index+"] "+fileSize(size));
}
}
public void logFailure(TransformReply reply)
{
RepositoryClientData repositoryClientData = new RepositoryClientData(reply.getClientData());