inteligr8/alfresco-transform-core
1
0
Fork 0
mirror of https://github.com/Alfresco/alfresco-transform-core.git synced 2025-08-14 17:58:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Save point: [skip ci]

Browse Source 
* probes and refactored 3 transform methods in TransformManager so they all use a TransformProcess class.
This commit is contained in:
alandavis
2022-07-19 18:21:31 +01:00
parent 87a5e8a480
commit 2763239538
21 changed files with 424 additions and 300 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
deprecated/alfresco-transformer-base/src/main/java/org/alfresco/transformer/AbstractTransformerController.java
View File

@@ -312,7 +312,7 @@ public abstract class AbstractTransformerController implements TransformControll
}
catch (TransformException e)
{
reply.setStatus(e.getStatusCode().value());
reply.setStatus(e.getStatusCode());
reply.setErrorDetails(messageWithCause("Failed at reading the source file", e));
transformerDebug.logFailure(reply);
@@ -356,7 +356,7 @@ public abstract class AbstractTransformerController implements TransformControll
}
catch (TransformException e)
{
reply.setStatus(e.getStatusCode().value());
reply.setStatus(e.getStatusCode());
reply.setErrorDetails(messageWithCause("Failed at processing transformation", e));
transformerDebug.logFailure(reply);
@@ -381,7 +381,7 @@ public abstract class AbstractTransformerController implements TransformControll
}
catch (TransformException e)
{
reply.setStatus(e.getStatusCode().value());
reply.setStatus(e.getStatusCode());
reply.setErrorDetails(messageWithCause("Failed at writing the transformed file", e));
transformerDebug.logFailure(reply);

2
deprecated/alfresco-transformer-base/src/main/java/org/alfresco/transformer/QueueTransformService.java
View File

@@ -116,7 +116,7 @@ public class QueueTransformService
catch (TransformException e)
{
logger.error(e.getMessage(), e);
replyWithError(replyToDestinationQueue, HttpStatus.valueOf(e.getStatusCode().value()),
replyWithError(replyToDestinationQueue, HttpStatus.valueOf(e.getStatusCode()),
e.getMessage(), correlationId);
return;
}

2
deprecated/alfresco-transformer-base/src/main/java/org/alfresco/transformer/TransformController.java
View File

@@ -210,7 +210,7 @@ public interface TransformController
TransformException e) throws IOException
{
final String message = e.getMessage();
final int statusCode = e.getStatusCode().value();
final int statusCode = e.getStatus().value();
logger.error(message, e);

18
engines/base/src/main/java/org/alfresco/transform/base/QueueTransformService.java
View File

@@ -36,6 +36,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.jms.annotation.JmsListener;
import org.springframework.jms.support.converter.MessageConversionException;
import org.springframework.stereotype.Component;
@@ -80,12 +81,12 @@ public class QueueTransformService
}
final String correlationId = tryRetrieveCorrelationId(msg);
Destination replyToDestinationQueue;
Destination replyToQueue;
try
{
replyToDestinationQueue = msg.getJMSReplyTo();
if (replyToDestinationQueue == null)
replyToQueue = msg.getJMSReplyTo();
if (replyToQueue == null)
{
logger.error(
"Cannot find 'replyTo' destination queue for message with correlationID {}. Stopping. ",
@@ -111,7 +112,7 @@ public class QueueTransformService
catch (TransformException e)
{
logger.error(e.getMessage(), e);
replyWithError(replyToDestinationQueue, HttpStatus.valueOf(e.getStatusCode().value()),
replyWithError(replyToQueue, HttpStatus.valueOf(e.getStatus().value()),
e.getMessage(), correlationId);
return;
}
@@ -119,13 +120,12 @@ public class QueueTransformService
if (!transformRequest.isPresent())
{
logger.error("T-Request from message with correlationID {} is null!", correlationId);
replyWithInternalSvErr(replyToDestinationQueue,
replyWithInternalSvErr(replyToQueue,
"JMS exception during T-Request deserialization: ", correlationId);
return;
}
TransformReply reply = transformHandler.handleMessageRequest(transformRequest.get(), null).getBody();
transformReplySender.send(replyToDestinationQueue, reply);
transformHandler.handleMessageRequest(transformRequest.get(), null, replyToQueue);
}
/**
@@ -172,7 +172,7 @@ public class QueueTransformService
replyWithError(destination, INTERNAL_SERVER_ERROR, msg, correlationId);
}
private void replyWithError(final Destination destination, final HttpStatus status,
private void replyWithError(final Destination replyToQueue, final HttpStatus status,
final String msg,
final String correlationId)
{
@@ -182,7 +182,7 @@ public class QueueTransformService
.withErrorDetails(msg)
.build();
transformReplySender.send(destination, reply, correlationId);
transformReplySender.send(replyToQueue, reply, correlationId);
}
private static String tryRetrieveCorrelationId(final Message msg)

16
engines/base/src/main/java/org/alfresco/transform/base/TransformController.java
View File

@@ -36,6 +36,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Controller;
import org.springframework.ui.Model;
@@ -172,7 +173,7 @@ public class TransformController
@ResponseBody
public String ready(HttpServletRequest request)
{
return probeTransform.doTransformOrNothing(request, false, this);
return probeTransform.doTransformOrNothing(request, false, transformHandler);
}
/**
@@ -182,7 +183,7 @@ public class TransformController
@ResponseBody
public String live(HttpServletRequest request)
{
return probeTransform.doTransformOrNothing(request, true, this);
return probeTransform.doTransformOrNothing(request, true, transformHandler);
}
@GetMapping(value = ENDPOINT_TRANSFORM_CONFIG)
@@ -253,11 +254,8 @@ public class TransformController
throws IOException
{
final String message = format("Request parameter ''{0}'' is missing", e.getParameterName());
final int statusCode = BAD_REQUEST.value();
logger.error(message, e);
LogEntry.setStatusCodeAndMessage(statusCode, message);
response.sendError(statusCode, message);
response.sendError(BAD_REQUEST.value(), message);
}
@ExceptionHandler(TransformException.class)
@@ -265,12 +263,8 @@ public class TransformController
throws IOException
{
final String message = e.getMessage();
final int statusCode = e.getStatusCode().value();
logger.error(message);
long time = LogEntry.setStatusCodeAndMessage(statusCode, message);
probeTransform.recordTransformTime(time);
response.sendError(statusCode, message);
response.sendError(e.getStatus().value(), message);
ModelAndView mav = new ModelAndView();
mav.addObject("title", transformEngine.getTransformEngineName() + " Error Page");

406
engines/base/src/main/java/org/alfresco/transform/base/TransformHandler.java
View File

@@ -27,11 +27,10 @@
package org.alfresco.transform.base;
import org.alfresco.transform.base.clients.AlfrescoSharedFileStoreClient;
import org.alfresco.transform.base.fs.FileManager;
import org.alfresco.transform.base.logging.LogEntry;
import org.alfresco.transform.base.messaging.TransformReplySender;
import org.alfresco.transform.base.model.FileRefResponse;
import org.alfresco.transform.base.probes.ProbeTransform;
import org.alfresco.transform.base.util.OutputStreamLengthRecorder;
import org.alfresco.transform.client.model.InternalContext;
import org.alfresco.transform.client.model.TransformReply;
import org.alfresco.transform.client.model.TransformRequest;
@@ -39,6 +38,7 @@ import org.alfresco.transform.common.ExtensionService;
import org.alfresco.transform.common.TransformException;
import org.alfresco.transform.common.TransformerDebug;
import org.alfresco.transform.messages.TransformRequestValidator;
import org.alfresco.transform.messages.TransformStack;
import org.alfresco.transform.registry.TransformServiceRegistry;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -56,13 +56,17 @@ import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;
import javax.annotation.PostConstruct;
import javax.jms.Destination;
import javax.servlet.http.HttpServletRequest;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Comparator;
@@ -74,6 +78,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import static java.util.stream.Collectors.joining;
import static org.alfresco.transform.base.fs.FileManager.createTargetFile;
import static org.alfresco.transform.base.fs.FileManager.getDirectAccessUrlInputStream;
import static org.alfresco.transform.base.fs.FileManager.getMultipartFileInputStream;
import static org.alfresco.transform.common.RequestParamMap.DIRECT_ACCESS_URL;
import static org.alfresco.transform.common.RequestParamMap.SOURCE_ENCODING;
import static org.alfresco.transform.common.RequestParamMap.SOURCE_EXTENSION;
@@ -107,6 +112,8 @@ public class TransformHandler
@Autowired
private TransformServiceRegistry transformRegistry;
@Autowired
private TransformReplySender transformReplySender;
@Autowired
private TransformerDebug transformerDebug;
private AtomicInteger httpRequestCount = new AtomicInteger(1);
@@ -174,155 +181,281 @@ public class TransformHandler
return probeTransform;
}
private abstract class TransformProcess
{
protected final String sourceMimetype;
protected final String targetMimetype;
private final Map<String, String> transformOptions;
protected String reference;
TransformProcess(String sourceMimetype, String targetMimetype, Map<String, String> transformOptions, String reference)
{
this.sourceMimetype = sourceMimetype;
this.targetMimetype = targetMimetype;
this.transformOptions = cleanTransformOptions(transformOptions);
this.reference = reference;
}
void handleTransformRequest()
{
LogEntry.start();
probeTransform.incrementTransformerCount();
TransformManagerImpl transformManager = TransformManagerImpl.builder()
.withSourceMimetype(sourceMimetype)
.withTargetMimetype(targetMimetype)
.build();
InputStream inputStream = null;
try
{
init(transformManager);
inputStream = getInputStream(transformManager);
long sourceSizeInBytes = getSourceSize();
String transformName = getTransformerName(sourceSizeInBytes, sourceMimetype, targetMimetype, transformOptions);
CustomTransformer customTransformer = getCustomTransformer(transformName);
transformerDebugPushTransform(sourceSizeInBytes, transformOptions, transformName);
OutputStream outputStream = getOutputStream(transformManager);
try
{
customTransformer.transform(sourceMimetype, inputStream,
targetMimetype, outputStream, transformOptions, transformManager);
transformManager.ifUsedCopyTargetFileToOutputStream();
sendTransformResponse(transformManager);
LogEntry.setTargetSize(transformManager.getOutputLength());
LogEntry.setStatusCodeAndMessage(OK, "Success");
}
finally
{
closeOutputStream(transformManager);
}
}
catch (TransformException e)
{
transformerDebug.logFailure(reference, e.getMessage());
LogEntry.setStatusCodeAndMessage(e.getStatus(), e.getMessage());
handleTransformException(e, e.getStatus());
}
catch (Exception e)
{
transformerDebug.logFailure(reference, e.getMessage());
LogEntry.setStatusCodeAndMessage(INTERNAL_SERVER_ERROR, e.getMessage());
handleException(e, INTERNAL_SERVER_ERROR);
}
finally
{
closeInputStreamWithoutException(inputStream);
deleteTmpFiles(transformManager);
long time = LogEntry.getTransformDuration();
probeTransform.recordTransformTime(time);
transformerDebug.popTransform(reference, time);
LogEntry.complete();
}
}
protected abstract void init(TransformManagerImpl transformManager);
protected abstract InputStream getInputStream(TransformManagerImpl transformManager);
protected abstract long getSourceSize();
protected void transformerDebugPushTransform(long sourceSizeInBytes, Map<String, String> transformOptions, String transformName)
{
transformerDebug.pushTransform(reference, sourceMimetype, targetMimetype, sourceSizeInBytes, transformName);
transformerDebug.logOptions(reference, transformOptions);
}
protected abstract OutputStream getOutputStream(TransformManagerImpl transformManager)
throws FileNotFoundException;
protected void sendTransformResponse(TransformManagerImpl transformManager)
{
// Only used in handleMessageRequest(...)
}
protected void closeOutputStream(TransformManagerImpl transformManager) throws IOException
{
transformManager.getOutputStream().close();
}
protected void handleTransformException(TransformException e, HttpStatus status)
{
throw e;
}
protected void handleException(Exception e, HttpStatus status)
{
throw new RuntimeException(e);
}
}
public ResponseEntity<StreamingResponseBody> handleHttpRequest(HttpServletRequest request,
MultipartFile sourceMultipartFile, String sourceMimetype, String targetMimetype,
Map<String, String> requestParameters)
{
return createResponseEntity(targetMimetype, os ->
{
TransformManagerImpl transformManager = null;
String reference = "e" + httpRequestCount.getAndIncrement();
try
new TransformProcess(sourceMimetype, targetMimetype, requestParameters,
"e" + httpRequestCount.getAndIncrement())
{
if (logger.isDebugEnabled())
@Override
protected void init(TransformManagerImpl transformManager)
{
logger.debug("Processing request via HTTP endpoint. Params: sourceMimetype: '{}', targetMimetype: '{}', "
+ "requestParameters: {}", sourceMimetype, targetMimetype, requestParameters);
transformManager.setRequest(request);
}
probeTransform.incrementTransformerCount();
final String directUrl = requestParameters.getOrDefault(DIRECT_ACCESS_URL, "");
InputStream inputStream = new BufferedInputStream(directUrl.isBlank() ?
FileManager.getMultipartFileInputStream(sourceMultipartFile) :
getDirectAccessUrlInputStream(directUrl));
long sourceSizeInBytes = -1L; // Ignore for http requests as the Alfresco repo will have checked.
Map<String, String> transformOptions = getTransformOptions(requestParameters);
String transformName = getTransformerName(sourceSizeInBytes, sourceMimetype, targetMimetype, transformOptions);
CustomTransformer customTransformer = getCustomTransformer(transformName);
transformerDebug.pushTransform(reference, sourceMimetype, targetMimetype, sourceSizeInBytes, transformName);
transformerDebug.logOptions(reference, requestParameters);
@Override
protected InputStream getInputStream(TransformManagerImpl transformManager)
{
return getInputStreamFromHttp(requestParameters, sourceMultipartFile, transformManager);
}
OutputStreamLengthRecorder outputStream = new OutputStreamLengthRecorder(os);
transformManager = TransformManagerImpl.builder()
.withRequest(request)
.withSourceMimetype(sourceMimetype)
.withTargetMimetype(targetMimetype)
.withInputStream(inputStream)
.withOutputStream(outputStream)
.build();
transformManager.setOutputStream(outputStream);
@Override
protected long getSourceSize()
{
return -1L; // Ignore for http requests as the Alfresco repo will have checked.
}
customTransformer.transform(sourceMimetype, inputStream,
targetMimetype, outputStream, transformOptions, transformManager);
@Override
protected OutputStream getOutputStream(TransformManagerImpl transformManager)
{
return transformManager.setOutputStream(os);
}
transformManager.ifUsedCopyTargetFileToOutputStream();
LogEntry.setTargetSize(outputStream.getLength());
long time = LogEntry.setStatusCodeAndMessage(OK.value(), "Success");
probeTransform.recordTransformTime(time);
transformerDebug.popTransform(reference, time);
}
catch (TransformException e)
{
transformerDebug.logFailure(reference, e.getMessage());
throw e;
}
catch (Exception e)
{
transformerDebug.logFailure(reference, e.getMessage());
throw new RuntimeException(e);
}
finally
{
deleteTmpFiles(transformManager);
}
@Override
protected void closeOutputStream(TransformManagerImpl transformManager) throws IOException
{
// Do nothing as the Spring mvc handles this for HttpServletRequest
}
}.handleTransformRequest();
});
}
public ResponseEntity<TransformReply> handleMessageRequest(TransformRequest request, Long timeout)
public void handleProbRequest(String sourceMimetype, String targetMimetype, Map<String, String> transformOptions,
File sourceFile, File targetFile)
{
long start = System.currentTimeMillis();
InputStream inputStream = null;
TransformManagerImpl transformManager = null;
TransformReply reply = createBasicTransformReply(request);;
try
new TransformProcess(sourceMimetype, targetMimetype, transformOptions,
"p" + httpRequestCount.getAndIncrement())
{
logger.trace("Received {}, timeout {} ms", request, timeout);
probeTransform.incrementTransformerCount();
checkTransformRequestValid(request, reply);
inputStream = getInputStream(request, reply);
String targetMimetype = request.getTargetMediaType();
String sourceMimetype = request.getSourceMediaType();
File targetFile = createTargetFile(null, sourceMimetype, targetMimetype);
transformerDebug.pushTransform(request);
long sourceSizeInBytes = request.getSourceSize();
Map<String, String> transformOptions = getTransformOptions(request.getTransformRequestOptions());
transformerDebug.logOptions(request);
String transformName = getTransformerName(sourceSizeInBytes, sourceMimetype, targetMimetype, transformOptions);
CustomTransformer customTransformer = getCustomTransformer(transformName);
try (OutputStreamLengthRecorder outputStream = new OutputStreamLengthRecorder(new BufferedOutputStream(
new FileOutputStream(targetFile))))
@Override
protected void init(TransformManagerImpl transformManager)
{
transformManager = TransformManagerImpl.builder()
.withSourceMimetype(sourceMimetype)
.withTargetMimetype(targetMimetype)
.withInputStream(inputStream)
.withOutputStream(outputStream)
.withTargetFile(targetFile)
.build();
customTransformer.transform(sourceMimetype, inputStream, targetMimetype, outputStream, transformOptions,
transformManager);
transformManager.ifUsedCopyTargetFileToOutputStream();
reply.getInternalContext().setCurrentSourceSize(outputStream.getLength());
saveTargetFileInSharedFileStore(targetFile, reply);
transformManager.setSourceFile(sourceFile);
transformManager.setTargetFile(targetFile);
transformManager.setSourceFileCreated();
// we don't want to delete the target file.
}
}
catch (TransformException e)
{
return createFailedResponseEntity(reply, e, e.getStatusCode().value());
}
catch (Exception e)
{
return createFailedResponseEntity(reply, e, INTERNAL_SERVER_ERROR.value());
}
finally
{
deleteTmpFiles(transformManager);
closeInputStreamWithoutException(inputStream);
probeTransform.recordTransformTime(System.currentTimeMillis()-start);
transformerDebug.popTransform(reply);
@Override
protected InputStream getInputStream(TransformManagerImpl transformManager)
{
return getInputStreamFromProb(sourceFile, transformManager);
}
logger.trace("Sending successful {}, timeout {} ms", reply, timeout);
return new ResponseEntity<>(reply, HttpStatus.valueOf(reply.getStatus()));
}
@Override
protected long getSourceSize()
{
return sourceFile.length();
}
@Override
protected OutputStream getOutputStream(TransformManagerImpl transformManager)
throws FileNotFoundException
{
return getOutputStreamFromFile(targetFile, transformManager);
}
}.handleTransformRequest();
}
private ResponseEntity<TransformReply> createFailedResponseEntity(TransformReply reply, Exception e,
int status) {
reply.setStatus(status);
public void handleMessageRequest(TransformRequest request, Long timeout, Destination replyToQueue)
{
new TransformProcess(request.getSourceMediaType(), request.getTargetMediaType(),
request.getTransformRequestOptions(), "unset")
{
TransformReply reply = createBasicTransformReply(request);
@Override
protected void init(TransformManagerImpl transformManager)
{
checkTransformRequestValid(request, reply);
reference = TransformStack.getReference(reply.getInternalContext());
}
@Override
protected InputStream getInputStream(TransformManagerImpl transformManager)
{
return getInputStreamFromMessage(request, transformManager);
}
@Override
protected void transformerDebugPushTransform(long sourceSizeInBytes, Map<String, String> transformOptions, String transformName)
{
transformerDebug.pushTransform(request);
transformerDebug.logOptions(request);
}
@Override
protected long getSourceSize()
{
return request.getSourceSize();
}
@Override
protected OutputStream getOutputStream(TransformManagerImpl transformManager)
throws FileNotFoundException
{
File targetFile = transformManager.setTargetFile(createTargetFile(null, sourceMimetype, targetMimetype));
transformManager.setTargetFileCreated();
return getOutputStreamFromFile(targetFile, transformManager);
}
protected void sendTransformResponse(TransformManagerImpl transformManager)
{
reply.getInternalContext().setCurrentSourceSize(transformManager.getOutputLength());
saveTargetFileInSharedFileStore(transformManager.getTargetFile(), reply);
sendSuccessfulResponse(timeout, reply, replyToQueue);
}
@Override
protected void handleTransformException(TransformException e, HttpStatus status)
{
sendFailedResponse(reply, e, status, replyToQueue);
}
@Override
protected void handleException(Exception e, HttpStatus status)
{
sendFailedResponse(reply, e, status, replyToQueue);
}
}.handleTransformRequest();
}
private void sendSuccessfulResponse(Long timeout, TransformReply reply, Destination replyToQueue)
{
logger.trace("Sending successful {}, timeout {} ms", reply, timeout);
transformReplySender.send(replyToQueue, reply);
}
private void sendFailedResponse(TransformReply reply, Exception e, HttpStatus status, Destination replyToQueue)
{
reply.setStatus(status.value());
reply.setErrorDetails(messageWithCause("Transform failed", e));
transformerDebug.logFailure(reply);
logger.trace("Transform failed. Sending " + reply, e);
return new ResponseEntity<>(reply, HttpStatus.valueOf(reply.getStatus()));
transformReplySender.send(replyToQueue, reply);
}
private void deleteTmpFiles(TransformManagerImpl transformManager)
{
if (transformManager != null)
{
transformManager.deleteSourceFileIfExists();
transformManager.deleteTargetFileIfExists();
transformManager.deleteSourceFileIfCreated();
transformManager.deleteTargetFileIfCreated();
}
}
@@ -373,7 +506,7 @@ public class TransformHandler
request.setInternalContext(InternalContext.initialise(request.getInternalContext()));
}
private Map<String, String> getTransformOptions(Map<String, String> requestParameters)
private Map<String, String> cleanTransformOptions(Map<String, String> requestParameters)
{
Map<String, String> transformOptions = new HashMap<>(requestParameters);
transformOptions.keySet().removeAll(NON_TRANSFORM_OPTION_REQUEST_PARAMETERS);
@@ -404,18 +537,39 @@ public class TransformHandler
}
}
private InputStream getInputStream(TransformRequest request, TransformReply reply)
private InputStream getInputStreamFromHttp(Map<String, String> requestParameters, MultipartFile sourceMultipartFile,
TransformManagerImpl transformManager)
{
final String directUrl = requestParameters.getOrDefault(DIRECT_ACCESS_URL, "");
return transformManager.setInputStream(new BufferedInputStream(directUrl.isBlank() ?
getMultipartFileInputStream(sourceMultipartFile) :
getDirectAccessUrlInputStream(directUrl)));
}
private InputStream getInputStreamFromProb(File sourceFile, TransformManagerImpl transformManager)
{
try
{
return transformManager.setInputStream(new BufferedInputStream(new FileInputStream(sourceFile)));
}
catch (FileNotFoundException e)
{
throw new TransformException(INTERNAL_SERVER_ERROR, messageWithCause("Failed to read the probe source", e));
}
}
private InputStream getInputStreamFromMessage(TransformRequest request, TransformManagerImpl transformManager)
{
final String directUrl = request.getTransformRequestOptions().getOrDefault(DIRECT_ACCESS_URL, "");
try
{
return new BufferedInputStream(directUrl.isBlank()
return transformManager.setInputStream(new BufferedInputStream(directUrl.isBlank()
? getSharedFileStoreInputStream(request.getSourceReference())
: getDirectAccessUrlInputStream(directUrl));
: getDirectAccessUrlInputStream(directUrl)));
}
catch (TransformException e)
{
throw new TransformException(e.getStatusCode(), messageWithCause("Failed to read the source", e));
throw new TransformException(e.getStatus(), messageWithCause("Failed to read the source", e));
}
catch (HttpClientErrorException e)
{
@@ -423,6 +577,12 @@ public class TransformHandler
}
}
private OutputStream getOutputStreamFromFile(File targetFile, TransformManagerImpl transformManager)
throws FileNotFoundException
{
return transformManager.setOutputStream(new BufferedOutputStream(new FileOutputStream(targetFile)));
}
private void saveTargetFileInSharedFileStore(File targetFile, TransformReply reply)
{
FileRefResponse targetRef;
@@ -432,7 +592,7 @@ public class TransformHandler
}
catch (TransformException e)
{
throw new TransformException(e.getStatusCode(), messageWithCause("Failed writing to SFS", e));
throw new TransformException(e.getStatus(), messageWithCause("Failed writing to SFS", e));
}
catch (HttpClientErrorException e)
{

10
engines/base/src/main/java/org/alfresco/transform/base/TransformInterceptor.java
View File

@@ -43,21 +43,11 @@ import org.springframework.web.servlet.handler.HandlerInterceptorAdapter;
*/
public class TransformInterceptor extends HandlerInterceptorAdapter
{
@Override
public boolean preHandle(HttpServletRequest request,
HttpServletResponse response, Object handler)
{
LogEntry.start();
return true;
}
@Override
public void afterCompletion(HttpServletRequest request,
HttpServletResponse response, Object handler, Exception ex)
{
deleteFile(request, SOURCE_FILE);
deleteFile(request, TARGET_FILE);
LogEntry.complete();
}
}

62
engines/base/src/main/java/org/alfresco/transform/base/TransformManagerImpl.java
View File

@@ -27,6 +27,7 @@
package org.alfresco.transform.base;
import org.alfresco.transform.base.fs.FileManager;
import org.alfresco.transform.base.util.OutputStreamLengthRecorder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
@@ -36,6 +37,9 @@ import java.io.File;
import java.io.InputStream;
import java.io.OutputStream;
/**
* Manages the input and output streams and any temporary files that have been created, which will need to be deleted.
*/
@Component
public class TransformManagerImpl implements TransformManager
{
@@ -43,13 +47,14 @@ public class TransformManagerImpl implements TransformManager
private HttpServletRequest request;
private InputStream inputStream;
private OutputStream outputStream;
private OutputStreamLengthRecorder outputStreamLengthRecorder;
private String sourceMimetype;
private String targetMimetype;
private File sourceFile;
private File targetFile;
private boolean createSourceFileCalled;
private boolean createTargetFileCalled;
private boolean sourceFileCreated;
private boolean targetFileCreated;
private TransformManagerImpl()
@@ -60,11 +65,12 @@ public class TransformManagerImpl implements TransformManager
{
request = null;
inputStream = null;
outputStream = null;
outputStreamLengthRecorder = null;
sourceFile = null;
targetFile = null;
createSourceFileCalled = false;
createTargetFileCalled = false;
sourceFileCreated = false;
targetFileCreated = false;
}
@@ -73,14 +79,26 @@ public class TransformManagerImpl implements TransformManager
this.request = request;
}
public void setInputStream(InputStream inputStream)
public InputStream setInputStream(InputStream inputStream)
{
this.inputStream = inputStream;
return inputStream;
}
public void setOutputStream(OutputStream outputStream)
public OutputStream getOutputStream()
{
this.outputStream = outputStream;
return outputStreamLengthRecorder;
}
public OutputStream setOutputStream(OutputStream outputStream)
{
this.outputStreamLengthRecorder = new OutputStreamLengthRecorder(outputStream);
return outputStream;
}
public Long getOutputLength()
{
return outputStreamLengthRecorder.getLength();
}
public void setSourceMimetype(String sourceMimetype)
@@ -107,19 +125,21 @@ public class TransformManagerImpl implements TransformManager
return targetFile;
}
void setTargetFile(File targetFile)
File setTargetFile(File targetFile)
{
this.targetFile = targetFile;
return targetFile;
}
public boolean isCreateSourceFileCalled()
public void setTargetFileCreated()
{
return createSourceFileCalled;
targetFileCreated = true;
}
public boolean isCreateTargetFileCalled()
public void setSourceFileCreated()
{
return createTargetFileCalled;
sourceFileCreated = true;
}
@Override public File createSourceFile()
@@ -133,6 +153,7 @@ public class TransformManagerImpl implements TransformManager
if (sourceFile == null)
{
sourceFile = FileManager.createSourceFile(request, inputStream, sourceMimetype);
sourceFileCreated = true;
}
return sourceFile;
}
@@ -157,22 +178,22 @@ public class TransformManagerImpl implements TransformManager
{
if (targetFileCreated)
{
FileManager.copyFileToOutputStream(targetFile, outputStream);
FileManager.copyFileToOutputStream(targetFile, outputStreamLengthRecorder);
}
}
public void deleteSourceFileIfExists()
public void deleteSourceFileIfCreated()
{
if (sourceFile != null && sourceFile.delete() == false)
if (sourceFile != null && sourceFileCreated && !sourceFile.delete())
{
logger.error("Failed to delete temporary source file "+sourceFile.getPath());
}
sourceFile = null;
}
public void deleteTargetFileIfExists()
public void deleteTargetFileIfCreated()
{
if (targetFile != null && targetFile.delete() == false)
if (targetFile != null && targetFileCreated && !targetFile.delete())
{
logger.error("Failed to delete temporary target file "+targetFile.getPath());
}
@@ -189,6 +210,7 @@ public class TransformManagerImpl implements TransformManager
" Fragments may only be sent with asynchronous requests. This a synchronous http request");
}
// TODO send the current output as a TransformResponse and then start a new one.
throw new UnsupportedOperationException("Not currently supported");
}
@@ -224,9 +246,9 @@ public class TransformManagerImpl implements TransformManager
return this;
}
public Builder withOutputStream(OutputStream outputStream)
public Builder withOutputStream(OutputStreamLengthRecorder outputStream)
{
transformManager.outputStream = outputStream;
transformManager.setOutputStream(outputStream);
return this;
}
@@ -236,6 +258,12 @@ public class TransformManagerImpl implements TransformManager
return this;
}
public Builder withSourceFile(File sourceFile)
{
transformManager.sourceFile = sourceFile;
return this;
}
public Builder withTargetFile(File targetFile)
{
transformManager.targetFile = targetFile;

10
engines/base/src/main/java/org/alfresco/transform/base/logging/LogEntry.java
View File

@@ -38,6 +38,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpStatus;
/**
* Provides setter and getter methods to allow the current Thread to set various log properties and for these
@@ -148,16 +149,21 @@ public final class LogEntry
currentLogEntry.get().options = options;
}
public static long setStatusCodeAndMessage(int statusCode, String message)
public static long setStatusCodeAndMessage(HttpStatus status, String message)
{
LogEntry logEntry = currentLogEntry.get();
logEntry.statusCode = statusCode;
logEntry.statusCode = status.value();
logEntry.message = message;
logEntry.durationTransform = System.currentTimeMillis() - logEntry.start - logEntry.durationStreamIn;
return logEntry.durationTransform;
}
public static long getTransformDuration()
{
return currentLogEntry.get().durationTransform;
}
public static void complete()
{
LogEntry logEntry = currentLogEntry.get();

6
engines/base/src/main/java/org/alfresco/transform/base/messaging/TransformReplySender.java
View File

@@ -56,8 +56,7 @@ public class TransformReplySender
send(destination, reply, reply.getRequestId());
}
public void send(final Destination destination, final TransformReply reply,
final String correlationId)
public void send(final Destination destination, final TransformReply reply, final String correlationId)
{
try
{
@@ -70,8 +69,7 @@ public class TransformReplySender
}
catch (Exception e)
{
logger.error(
"Failed to send T-Reply " + reply + " - for correlation ID " + correlationId, e);
logger.error("Failed to send T-Reply " + reply + " - for correlation ID " + correlationId, e);
}
}
}

77
engines/base/src/main/java/org/alfresco/transform/base/probes/ProbeTransform.java
View File

@@ -26,33 +26,28 @@
*/
package org.alfresco.transform.base.probes;
import static org.alfresco.transform.base.fs.FileManager.SOURCE_FILE;
import static org.alfresco.transform.base.fs.FileManager.TARGET_FILE;
import static org.alfresco.transform.base.fs.FileManager.TempFileProvider.createTempFile;
import static org.springframework.http.HttpStatus.BAD_REQUEST;
import static org.springframework.http.HttpStatus.INSUFFICIENT_STORAGE;
import static org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR;
import static org.springframework.http.HttpStatus.OK;
import static org.springframework.http.HttpStatus.TOO_MANY_REQUESTS;
import org.alfresco.transform.base.TransformHandler;
import org.alfresco.transform.base.logging.LogEntry;
import org.alfresco.transform.common.TransformException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.servlet.http.HttpServletRequest;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import javax.servlet.http.HttpServletRequest;
import org.alfresco.transform.base.TransformController;
import org.alfresco.transform.common.TransformException;
import org.alfresco.transform.registry.TransformServiceRegistry;
import org.alfresco.transform.base.logging.LogEntry;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import static org.alfresco.transform.base.fs.FileManager.TempFileProvider.createTempFile;
import static org.springframework.http.HttpStatus.INSUFFICIENT_STORAGE;
import static org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR;
import static org.springframework.http.HttpStatus.OK;
import static org.springframework.http.HttpStatus.TOO_MANY_REQUESTS;
/**
* Provides test transformations and the logic used by k8 liveness and readiness probes.
@@ -85,9 +80,6 @@ public class ProbeTransform
{
private final Logger logger = LoggerFactory.getLogger(ProbeTransform.class);
@Autowired
private TransformServiceRegistry transformRegistry;
private static final int AVERAGE_OVER_TRANSFORMS = 5;
private final String sourceFilename;
private final String targetFilename;
@@ -133,7 +125,7 @@ public class ProbeTransform
this.targetFilename = targetFilename;
this.sourceMimetype = sourceMimetype;
this.targetMimetype = targetMimetype;
this.transformOptions = transformOptions;
this.transformOptions = new HashMap(transformOptions);
minExpectedLength = Math.max(0, expectedLength - plusOrMinus);
maxExpectedLength = expectedLength + plusOrMinus;
@@ -180,7 +172,8 @@ public class ProbeTransform
}
// We don't want to be doing test transforms every few seconds, but do want frequent live probes.
public String doTransformOrNothing(HttpServletRequest request, boolean isLiveProbe, TransformController controller)
public String doTransformOrNothing(HttpServletRequest request, boolean isLiveProbe,
TransformHandler transformHandler)
{
// If not initialised OR it is a live probe and we are scheduled to to do a test transform.
probeCount++;
@@ -192,7 +185,7 @@ public class ProbeTransform
return (isLiveProbe && livenessTransformPeriod > 0 &&
(transCount <= AVERAGE_OVER_TRANSFORMS || nextTransformTime < System.currentTimeMillis()))
|| !initialised.get()
? doTransform(request, isLiveProbe, controller)
? doTransform(isLiveProbe, transformHandler)
: doNothing(isLiveProbe);
}
@@ -200,7 +193,6 @@ public class ProbeTransform
{
String probeMessage = getProbeMessage(isLiveProbe);
String message = "Success - No transform.";
LogEntry.setStatusCodeAndMessage(OK.value(), probeMessage + message);
if (!isLiveProbe && !readySent.getAndSet(true))
{
logger.trace("{}{}", probeMessage, message);
@@ -208,7 +200,7 @@ public class ProbeTransform
return message;
}
private String doTransform(HttpServletRequest request, boolean isLiveProbe, TransformController controller)
private String doTransform(boolean isLiveProbe, TransformHandler transformHandler)
{
checkMaxTransformTimeAndCount(isLiveProbe);
@@ -223,12 +215,9 @@ public class ProbeTransform
while (nextTransformTime < start);
}
File sourceFile = getSourceFile(request, isLiveProbe);
File targetFile = getTargetFile(request);
String transformName = getTransformerName(sourceFile, sourceMimetype, targetMimetype, transformOptions);
// controller.transformImpl(transformName, sourceMimetype, targetMimetype, transformOptions, sourceFile, targetFile);
File sourceFile = getSourceFile(isLiveProbe);
File targetFile = getTargetFile();
transformHandler.handleProbRequest(sourceMimetype, targetMimetype, transformOptions, sourceFile, targetFile);
long time = System.currentTimeMillis() - start;
String message = "Transform " + time + "ms";
checkTargetFile(targetFile, isLiveProbe, message);
@@ -249,20 +238,7 @@ public class ProbeTransform
checkMaxTransformTimeAndCount(isLiveProbe);
return getProbeMessage(isLiveProbe) + message;
}
private String getTransformerName(final File sourceFile, final String sourceMimetype,
final String targetMimetype, final Map<String, String> transformOptions)
{
final long sourceSizeInBytes = sourceFile.length();
final String transformerName = transformRegistry.findTransformerName(sourceMimetype,
sourceSizeInBytes, targetMimetype, transformOptions, null);
if (transformerName == null)
{
throw new TransformException(BAD_REQUEST, "No transforms were able to handle the request");
}
return transformerName;
return getProbeMessage(isLiveProbe) + "Success - "+message;
}
private void checkMaxTransformTimeAndCount(boolean isLiveProbe)
@@ -282,12 +258,11 @@ public class ProbeTransform
}
}
private File getSourceFile(HttpServletRequest request, boolean isLiveProbe)
private File getSourceFile(boolean isLiveProbe)
{
incrementTransformerCount();
File sourceFile = createTempFile("source_", "_" + sourceFilename);
request.setAttribute(SOURCE_FILE, sourceFile);
try (InputStream inputStream = this.getClass().getResourceAsStream('/' + sourceFilename))
try (InputStream inputStream = getClass().getResourceAsStream('/' + sourceFilename))
{
Files.copy(inputStream, sourceFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
}
@@ -301,10 +276,9 @@ public class ProbeTransform
return sourceFile;
}
private File getTargetFile(HttpServletRequest request)
private File getTargetFile()
{
File targetFile = createTempFile("target_", "_" + targetFilename);
request.setAttribute(TARGET_FILE, targetFile);
LogEntry.setTarget(targetFilename);
return targetFile;
}
@@ -353,6 +327,7 @@ public class ProbeTransform
probeMessage + "Target File \"" + targetFile.getAbsolutePath() + "\" did not exist");
}
long length = targetFile.length();
targetFile.delete();
if (length < minExpectedLength || length > maxExpectedLength)
{
throw new TransformException(INTERNAL_SERVER_ERROR,
@@ -360,8 +335,6 @@ public class ProbeTransform
"\" was the wrong size (" + length + "). Needed to be between " +
minExpectedLength + " and " + maxExpectedLength);
}
LogEntry.setTargetSize(length);
LogEntry.setStatusCodeAndMessage(OK.value(), probeMessage + "Success - " + message);
}
private String getMessagePrefix(boolean isLiveProbe)

8
engines/base/src/test/java/org/alfresco/transform/base/QueueTransformServiceTest.java
View File

@@ -188,12 +188,12 @@ public class QueueTransformServiceTest
doReturn(request).when(transformMessageConverter).fromMessage(msg);
doReturn(new ResponseEntity<>(reply, HttpStatus.valueOf(reply.getStatus())))
.when(transformHandler).handleMessageRequest(request, null);
.when(transformHandler).handleMessageRequest(request, null, null);
queueTransformService.receive(msg);
verify(transformMessageConverter).fromMessage(msg);
verify(transformHandler).handleMessageRequest(request, null);
verify(transformHandler).handleMessageRequest(request, null, null);
verify(transformReplySender).send(destination, reply);
}
@@ -229,12 +229,12 @@ public class QueueTransformServiceTest
doReturn(request).when(transformMessageConverter).fromMessage(msg);
doReturn(new ResponseEntity<>(reply, HttpStatus.valueOf(reply.getStatus())))
.when(transformHandler).handleMessageRequest(request, null);
.when(transformHandler).handleMessageRequest(request, null, null);
queueTransformService.receive(msg);
verify(transformMessageConverter).fromMessage(msg);
verify(transformHandler).handleMessageRequest(request, null);
verify(transformHandler).handleMessageRequest(request, null, null);
verify(transformReplySender).send(destination, reply);
}
}

14
engines/base/src/test/java/org/alfresco/transform/base/components/AbstractTestTransformEngine.java
View File

@@ -26,19 +26,11 @@
*/
package org.alfresco.transform.base.components;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import org.alfresco.transform.base.TransformEngine;
import org.alfresco.transform.base.probes.ProbeTransform;
import org.alfresco.transform.config.SupportedSourceAndTarget;
import org.alfresco.transform.config.TransformConfig;
import org.alfresco.transform.config.TransformOptionValue;
import org.alfresco.transform.config.Transformer;
import org.springframework.boot.test.context.TestComponent;
import static org.alfresco.transform.common.Mimetype.MIMETYPE_HTML;
import static org.alfresco.transform.common.Mimetype.MIMETYPE_IMAGE_PNG;
import static org.alfresco.transform.common.Mimetype.MIMETYPE_PDF;
import static org.alfresco.transform.common.Mimetype.MIMETYPE_TEXT_PLAIN;
import static org.alfresco.transform.common.RequestParamMap.SOURCE_ENCODING;
@@ -64,9 +56,9 @@ public abstract class AbstractTestTransformEngine implements TransformEngine
@Override public ProbeTransform getProbeTransform()
{
return new ProbeTransform("quick.html", "quick.txt",
MIMETYPE_HTML, MIMETYPE_TEXT_PLAIN, ImmutableMap.of(SOURCE_ENCODING, "UTF-8"),
119, 30, 150, 1024, 60 * 2 + 1,
return new ProbeTransform("quick.txt", "quick.pdf",
MIMETYPE_TEXT_PLAIN, MIMETYPE_PDF, ImmutableMap.of(SOURCE_ENCODING, "UTF-8"),
46, 0, 150, 1024, 1,
60 * 2);
}
}

2
engines/base/src/test/java/org/alfresco/transform/base/components/TestTransformEngineTwoTransformers.java
View File

@@ -82,7 +82,7 @@ public class TestTransformEngineTwoTransformers extends AbstractTestTransformEng
.withSupportedSourceAndTargetList(ImmutableSet.of(
SupportedSourceAndTarget.builder()
.withSourceMediaType(MIMETYPE_TEXT_PLAIN)
.withTargetMediaType(MIMETYPE_PDF)
.withTargetMediaType(MIMETYPE_IMAGE_PNG)
.build()))
.withTransformOptions(ImmutableSet.of(imageOptions))
.build(),

17
engines/base/src/test/resources/quick.html
View File

@@ -1,17 +0,0 @@
<html>
<head>
<meta http-equiv=Content-Type content="text/html; charset=windows-1252">
<title>The quick brown fox jumps over the lazy dog</title>
<meta name="author" content="Nevin Nollop">
<meta name="keywords" content="Pangram, fox, dog">
<meta name="description" content="Gym class featuring a brown fox and lazy dog">
</head>
<body lang=EN-US>
The quick brown fox jumps over the lazy dog
</body>
</html>

1
engines/base/src/test/resources/quick.txt Normal file
View File

@@ -0,0 +1 @@
Original Text

1
engines/base/src/test/resources/quick2.pdf Normal file
View File

@@ -0,0 +1 @@
Original Text -> TxT2Pdf()

18
engines/base/src/test/resources/quick2.txt
View File

@@ -1,18 +0,0 @@
The quick brown fox jumps over the lazy dog
The quick brown fox jumps over the lazy dog

2
engines/tika/src/test/java/org/alfresco/transform/tika/TikaTest.java
View File

@@ -267,7 +267,7 @@ public class TikaTest extends AbstractBaseTest
// See ACS-373
ProbeTransform probeTransform = getProbeTestTransform();
ReflectionTestUtils.setField(probeTransform, "livenessTransformEnabled", true);
probeTransform.doTransformOrNothing(httpServletRequest, true, controller);
probeTransform.doTransformOrNothing(httpServletRequest, true, transformHandler);
}
@Test

30
model/src/main/java/org/alfresco/transform/common/TransformException.java
View File

@@ -25,22 +25,40 @@ import org.springframework.http.HttpStatus;
public class TransformException extends RuntimeException
{
private final HttpStatus statusCode;
private final HttpStatus status;
public TransformException(HttpStatus statusCode, String message)
{
super(message);
this.statusCode = statusCode;
this.status = statusCode;
}
public TransformException(HttpStatus statusCode, String message, Throwable cause)
public TransformException(HttpStatus status, String message, Throwable cause)
{
super(message, cause);
this.statusCode = statusCode;
this.status = status;
}
public HttpStatus getStatusCode()
@Deprecated
public TransformException(int statusCode, String message)
{
return statusCode;
this(HttpStatus.valueOf(statusCode), message);
}
@Deprecated
public TransformException(int statusCode, String message, Throwable cause)
{
this(HttpStatus.valueOf(statusCode), message, cause);
}
@Deprecated
public int getStatusCode()
{
return status.value();
}
public HttpStatus getStatus()
{
return status;
}
}

16
model/src/main/java/org/alfresco/transform/common/TransformerDebug.java
View File

@@ -104,7 +104,14 @@ public class TransformerDebug
InternalContext internalContext = reply.getInternalContext();
String reference = TransformStack.getReference(internalContext);
long elapsedTime = TransformStack.getElapsedTime(internalContext);
popTransform(reference, elapsedTime);
}
}
public void popTransform(String reference, long elapsedTime)
{
if (logger.isDebugEnabled())
{
String message = getPaddedReference(reference) + "Finished in " + ms(elapsedTime);
if (isTopLevel(reference) || isTEngine())
{
@@ -118,15 +125,6 @@ public class TransformerDebug
}
}
public void popTransform(String reference, long elapsedTime)
{
if (logger.isDebugEnabled())
{
String message = getPaddedReference(reference) + "Finished in " + ms(elapsedTime);
logger.debug(message);
}
}
public void logOptions(TransformRequest request)
{
RepositoryClientData repositoryClientData = new RepositoryClientData(request.getClientData());