ATS-175 : T-Engine code cleanup

This commit is contained in:
Cezar.Leahu
2018-10-25 18:21:50 +03:00
parent ba8707c762
commit d85c03d362
42 changed files with 2042 additions and 1475 deletions

View File

@@ -12,6 +12,8 @@
package org.alfresco.transformer;
import io.micrometer.core.instrument.MeterRegistry;
import org.alfresco.transformer.executors.ImageMagickCommandExecutor;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.actuate.autoconfigure.metrics.MeterRegistryCustomizer;
@@ -27,9 +29,15 @@ public class Application
@Value("${container.name}")
private String containerName;
@Bean MeterRegistryCustomizer<MeterRegistry> metricsCommonTags() {
@Bean
MeterRegistryCustomizer<MeterRegistry> metricsCommonTags() {
return registry -> registry.config().commonTags("containerName", containerName);
}
@Bean
public ImageMagickCommandExecutor commandExecutor() {
return new ImageMagickCommandExecutor();
}
public static void main(String[] args)
{

View File

@@ -11,16 +11,27 @@
*/
package org.alfresco.transformer;
import static org.alfresco.transformer.fs.FileManager.createAttachment;
import static org.alfresco.transformer.fs.FileManager.createSourceFile;
import static org.alfresco.transformer.fs.FileManager.createTargetFile;
import static org.alfresco.transformer.fs.FileManager.createTargetFileName;
import static org.alfresco.transformer.logging.StandardMessages.ENTERPRISE_LICENCE;
import static org.alfresco.transformer.util.Util.stringToBoolean;
import static org.alfresco.transformer.util.Util.stringToInteger;
import java.io.File;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.StringJoiner;
import javax.servlet.http.HttpServletRequest;
import org.alfresco.util.exec.RuntimeExec;
import org.alfresco.transformer.exceptions.TransformException;
import org.alfresco.transformer.executors.ImageMagickCommandExecutor;
import org.alfresco.transformer.logging.LogEntry;
import org.alfresco.transformer.probes.ProbeTestTransform;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.Resource;
@@ -56,73 +67,46 @@ import org.springframework.web.multipart.MultipartFile;
@Controller
public class ImageMagickController extends AbstractTransformerController
{
private static final String ROOT = "/usr/lib64/ImageMagick-7.0.7";
private static final String DYN = ROOT+"/lib";
private static final String EXE = "/usr/bin/convert";
private static final Log logger = LogFactory.getLog(ImageMagickController.class);
private static final List<String> GRAVITY_VALUES = Arrays.asList(
"North", "NorthEast", "East", "SouthEast", "South", "SouthWest", "West", "NorthWest", "Center");
@Autowired
private ImageMagickCommandExecutor commandExecutor;
@Autowired
public ImageMagickController()
{
logger = LogFactory.getLog(ImageMagickController.class);
logger.info("--------------------------------------------------------------------------------------------------------------------------------------------------------------");
logEnterpriseLicenseMessage();
Arrays.stream(ENTERPRISE_LICENCE.split("\\n")).forEach(logger::info);
logger.info("This transformer uses ImageMagick from ImageMagick Studio LLC. See the license at http://www.imagemagick.org/script/license.php or in /ImageMagick-license.txt");
logger.info("--------------------------------------------------------------------------------------------------------------------------------------------------------------");
setTransformCommand(createTransformCommand());
setCheckCommand(createCheckCommand());
}
@Override
protected String getTransformerName()
public String getTransformerName()
{
return "ImageMagick";
}
private static RuntimeExec createTransformCommand()
@Override
public String version()
{
RuntimeExec runtimeExec = new RuntimeExec();
Map<String, String[]> commandsAndArguments = new HashMap<>();
commandsAndArguments.put(".*", new String[]{EXE, "${source}", "SPLIT:${options}", "-strip", "-quiet", "${target}"});
runtimeExec.setCommandsAndArguments(commandsAndArguments);
Map<String, String> processProperties = new HashMap<>();
processProperties.put("MAGICK_HOME", ROOT);
processProperties.put("DYLD_FALLBACK_LIBRARY_PATH", DYN);
processProperties.put("LD_LIBRARY_PATH", DYN);
runtimeExec.setProcessProperties(processProperties);
Map<String, String> defaultProperties = new HashMap<>();
defaultProperties.put("options", null);
runtimeExec.setDefaultProperties(defaultProperties);
runtimeExec.setErrorCodes("1,2,255,400,405,410,415,420,425,430,435,440,450,455,460,465,470,475,480,485,490,495,499,700,705,710,715,720,725,730,735,740,750,755,760,765,770,775,780,785,790,795,799");
return runtimeExec;
}
private static RuntimeExec createCheckCommand()
{
RuntimeExec runtimeExec = new RuntimeExec();
Map<String, String[]> commandsAndArguments = new HashMap<>();
commandsAndArguments.put(".*", new String[]{EXE, "-version"});
runtimeExec.setCommandsAndArguments(commandsAndArguments);
return runtimeExec;
return commandExecutor.version();
}
@Override
protected ProbeTestTransform getProbeTestTransform()
public ProbeTestTransform getProbeTestTransform()
{
// See the Javadoc on this method and Probes.md for the choice of these values.
return new ProbeTestTransform(this, "quick.jpg", "quick.png",
35593, 1024, 150, 1024, 60*15+1,60*15+0)
return new ProbeTestTransform(this, logger, "quick.jpg", "quick.png",
35593, 1024, 150, 1024, 60*15+1,60*15)
{
@Override
protected void executeTransformCommand(File sourceFile, File targetFile)
{
ImageMagickController.this.executeTransformCommand("", sourceFile, "", targetFile, null);
commandExecutor.run("", sourceFile, "", targetFile, null);
}
};
}
@@ -168,6 +152,7 @@ public class ImageMagickController extends AbstractTransformerController
{
String targetFilename = createTargetFileName(sourceMultipartFile.getOriginalFilename(),
targetExtension);
getProbeTestTransform().incrementTransformerCount();
File sourceFile = createSourceFile(request, sourceMultipartFile);
File targetFile = createTargetFile(request, targetFilename);
// Both files are deleted by TransformInterceptor.afterCompletion
@@ -176,14 +161,20 @@ public class ImageMagickController extends AbstractTransformerController
cropXOffset, cropYOffset, thumbnail, resizeWidth, resizeHeight, resizePercentage, allowEnlargement, maintainAspectRatio, commandOptions);
String pageRange = calculatePageRange(startPage, endPage);
executeTransformCommand(options, sourceFile, pageRange, targetFile, timeout);
commandExecutor.run(options, sourceFile, pageRange, targetFile,
timeout);
return createAttachment(targetFilename, targetFile, testDelay);
final ResponseEntity<Resource> body = createAttachment(targetFilename, targetFile);
LogEntry.setTargetSize(targetFile.length());
long time = LogEntry.setStatusCodeAndMessage(200, "Success");
time += LogEntry.addDelay(testDelay);
getProbeTestTransform().recordTransformTime(time);
return body;
}
@Override
protected void processTransform(File sourceFile, File targetFile,
Map<String, String> transformOptions, Long timeout)
public void processTransform(final File sourceFile, final File targetFile,
final Map<String, String> transformOptions, final Long timeout)
{
Integer startPage = stringToInteger(transformOptions.get("startPage"));
Integer endPage = stringToInteger(transformOptions.get("endPage"));
@@ -201,28 +192,17 @@ public class ImageMagickController extends AbstractTransformerController
Boolean resizePercentage = stringToBoolean(transformOptions.get("resizePercentage"));
Boolean allowEnlargement = stringToBoolean(transformOptions.get("allowEnlargement"));
Boolean maintainAspectRatio = stringToBoolean(transformOptions.get("maintainAspectRatio"));
String commandOptions = transformOptions.get("commandOptions");
String options = buildTransformOptions(startPage, endPage , alphaRemove, autoOrient, cropGravity, cropWidth, cropHeight, cropPercentage,
cropXOffset, cropYOffset, thumbnail, resizeWidth, resizeHeight, resizePercentage, allowEnlargement, maintainAspectRatio, commandOptions);
String pageRange = calculatePageRange(startPage, endPage);
final String options = buildTransformOptions(startPage, endPage, alphaRemove, autoOrient,
cropGravity, cropWidth, cropHeight, cropPercentage,
cropXOffset, cropYOffset, thumbnail, resizeWidth, resizeHeight, resizePercentage, allowEnlargement, maintainAspectRatio, null);
final String pageRange = calculatePageRange(startPage, endPage);
executeTransformCommand(options, sourceFile, pageRange, targetFile, timeout);
commandExecutor.run(options, sourceFile, pageRange, targetFile,
timeout);
}
private void executeTransformCommand(String options, File sourceFile, String pageRange, File targetFile, Long timeout)
{
LogEntry.setOptions(pageRange+(pageRange.isEmpty() ? "" : " ")+options);
Map<String, String> properties = new HashMap<String, String>(5);
properties.put("options", options);
properties.put("source", sourceFile.getAbsolutePath()+pageRange);
properties.put("target", targetFile.getAbsolutePath());
executeTransformCommand(properties, targetFile, timeout);
}
private String buildTransformOptions(Integer startPage, Integer endPage, Boolean alphaRemove,
private static String buildTransformOptions(Integer startPage, Integer endPage, Boolean alphaRemove,
Boolean autoOrient, String cropGravity, Integer cropWidth, Integer cropHeight,
Boolean cropPercentage, Integer cropXOffset, Integer cropYOffset, Boolean thumbnail,
Integer resizeWidth, Integer resizeHeight, Boolean resizePercentage,
@@ -261,7 +241,7 @@ public class ImageMagickController extends AbstractTransformerController
args.add(cropGravity);
}
StringBuilder crop = new StringBuilder("");
StringBuilder crop = new StringBuilder();
if (cropWidth != null && cropWidth >= 0)
{
crop.append(cropWidth);
@@ -303,7 +283,7 @@ public class ImageMagickController extends AbstractTransformerController
if (resizeHeight != null || resizeWidth != null || resizePercentage !=null || maintainAspectRatio != null)
{
args.add(thumbnail != null && thumbnail ? "-thumbnail" : "-resize");
StringBuilder resize = new StringBuilder("");
StringBuilder resize = new StringBuilder();
if (resizeWidth != null && resizeWidth >= 0)
{
resize.append(resizeWidth);
@@ -335,10 +315,9 @@ public class ImageMagickController extends AbstractTransformerController
args.toString();
}
private String calculatePageRange(Integer startPage, Integer endPage)
private static String calculatePageRange(Integer startPage, Integer endPage)
{
return
startPage == null
return startPage == null
? endPage == null
? ""
: "["+endPage+']'

View File

@@ -0,0 +1,52 @@
package org.alfresco.transformer.executors;
import java.util.HashMap;
import java.util.Map;
import org.alfresco.util.exec.RuntimeExec;
import org.springframework.stereotype.Component;
/**
*/
@Component
public class ImageMagickCommandExecutor extends AbstractCommandExecutor
{
private static final String ROOT = "/usr/lib64/ImageMagick-7.0.7";
private static final String DYN = ROOT + "/lib";
private static final String EXE = "/usr/bin/convert";
@Override
protected RuntimeExec createTransformCommand()
{
RuntimeExec runtimeExec = new RuntimeExec();
Map<String, String[]> commandsAndArguments = new HashMap<>();
commandsAndArguments.put(".*",
new String[]{EXE, "${source}", "SPLIT:${options}", "-strip", "-quiet", "${target}"});
runtimeExec.setCommandsAndArguments(commandsAndArguments);
Map<String, String> processProperties = new HashMap<>();
processProperties.put("MAGICK_HOME", ROOT);
processProperties.put("DYLD_FALLBACK_LIBRARY_PATH", DYN);
processProperties.put("LD_LIBRARY_PATH", DYN);
runtimeExec.setProcessProperties(processProperties);
Map<String, String> defaultProperties = new HashMap<>();
defaultProperties.put("options", null);
runtimeExec.setDefaultProperties(defaultProperties);
runtimeExec.setErrorCodes(
"1,2,255,400,405,410,415,420,425,430,435,440,450,455,460,465,470,475,480,485,490,495,499,700,705,710,715,720,725,730,735,740,750,755,760,765,770,775,780,785,790,795,799");
return runtimeExec;
}
@Override
protected RuntimeExec createCheckCommand()
{
RuntimeExec runtimeExec = new RuntimeExec();
Map<String, String[]> commandsAndArguments = new HashMap<>();
commandsAndArguments.put(".*", new String[]{EXE, "-version"});
runtimeExec.setCommandsAndArguments(commandsAndArguments);
return runtimeExec;
}
}

View File

@@ -25,21 +25,48 @@
*/
package org.alfresco.transformer;
import static org.hamcrest.Matchers.containsString;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.when;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
import org.alfresco.transform.client.model.TransformReply;
import org.alfresco.transform.client.model.TransformRequest;
import org.alfresco.transformer.executors.ImageMagickCommandExecutor;
import org.alfresco.transformer.model.FileRefEntity;
import org.alfresco.transformer.model.FileRefResponse;
import org.alfresco.util.exec.RuntimeExec;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.stubbing.Answer;
import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest;
import org.springframework.boot.test.mock.mockito.SpyBean;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.Resource;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.mock.web.MockMultipartFile;
import org.springframework.test.context.junit4.SpringRunner;
import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
import org.springframework.util.StringUtils;
/**
* Test the ImageMagickController without a server.
@@ -49,16 +76,104 @@ import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
@WebMvcTest(ImageMagickController.class)
public class ImageMagickControllerTest extends AbstractTransformerControllerTest
{
@Mock
private RuntimeExec.ExecutionResult mockExecutionResult;
@Mock
private RuntimeExec mockTransformCommand;
@Mock
private RuntimeExec mockCheckCommand;
@SpyBean
private ImageMagickCommandExecutor commandExecutor;
@SpyBean
private ImageMagickController controller;
@Before
public void before() throws IOException
{
controller.setAlfrescoSharedFileStoreClient(alfrescoSharedFileStoreClient);
super.controller = controller;
commandExecutor.setTransformCommand(mockTransformCommand);
commandExecutor.setCheckCommand(mockCheckCommand);
super.mockTransformCommand(controller, "jpg", "png", "image/jpg", true);
mockTransformCommand("jpg", "png", "image/jpg", true);
}
@Override
protected void mockTransformCommand(String sourceExtension,
String targetExtension, String sourceMimetype,
boolean readTargetFileBytes) throws IOException
{
this.sourceExtension = sourceExtension;
this.targetExtension = targetExtension;
this.sourceMimetype = sourceMimetype;
expectedOptions = null;
expectedSourceSuffix = null;
expectedSourceFileBytes = readTestFile(sourceExtension);
expectedTargetFileBytes = readTargetFileBytes ? readTestFile(targetExtension) : null;
sourceFile = new MockMultipartFile("file", "quick." + sourceExtension, sourceMimetype, expectedSourceFileBytes);
when(mockTransformCommand.execute(any(), anyLong())).thenAnswer(
(Answer<RuntimeExec.ExecutionResult>) invocation -> {
Map<String, String> actualProperties = invocation.getArgument(0);
assertEquals("There should be 3 properties", 3, actualProperties.size());
String actualOptions = actualProperties.get("options");
String actualSource = actualProperties.get("source");
String actualTarget = actualProperties.get("target");
String actualTargetExtension = StringUtils.getFilenameExtension(actualTarget);
assertNotNull(actualSource);
assertNotNull(actualTarget);
if (expectedSourceSuffix != null)
{
assertTrue("The source file \""+actualSource+"\" should have ended in \""+expectedSourceSuffix+"\"", actualSource.endsWith(expectedSourceSuffix));
actualSource = actualSource.substring(0, actualSource.length()-expectedSourceSuffix.length());
}
assertNotNull(actualOptions);
if (expectedOptions != null)
{
assertEquals("expectedOptions", expectedOptions, actualOptions);
}
Long actualTimeout = invocation.getArgument(1);
assertNotNull(actualTimeout);
if (expectedTimeout != null)
{
assertEquals("expectedTimeout", expectedTimeout, actualTimeout);
}
// Copy a test file into the target file location if it exists
int i = actualTarget.lastIndexOf('_');
if (i >= 0)
{
String testFilename = actualTarget.substring(i+1);
File testFile = getTestFile(testFilename, false);
File targetFile = new File(actualTarget);
generateTargetFileFromResourceFile(actualTargetExtension, testFile,
targetFile);
}
// Check the supplied source file has not been changed.
byte[] actualSourceFileBytes = Files.readAllBytes(new File(actualSource).toPath());
assertTrue("Source file is not the same", Arrays.equals(expectedSourceFileBytes, actualSourceFileBytes));
return mockExecutionResult;
});
when(mockExecutionResult.getExitValue()).thenReturn(0);
when(mockExecutionResult.getStdErr()).thenReturn("STDERROR");
when(mockExecutionResult.getStdOut()).thenReturn("STDOUT");
}
@Override
protected AbstractTransformerController getController()
{
return controller;
}
@Test
@@ -67,7 +182,7 @@ public class ImageMagickControllerTest extends AbstractTransformerControllerTest
for (String value: new String[] {"North", "NorthEast", "East", "SouthEast", "South", "SouthWest", "West", "NorthWest", "Center"})
{
expectedOptions = "-gravity "+value+" +repage";
mockMvc.perform(MockMvcRequestBuilders.fileUpload("/transform")
mockMvc.perform(MockMvcRequestBuilders.multipart("/transform")
.file(sourceFile)
.param("targetExtension", targetExtension)
.param("cropGravity", value))
@@ -80,7 +195,7 @@ public class ImageMagickControllerTest extends AbstractTransformerControllerTest
@Test
public void cropGravityBadTest() throws Exception
{
mockMvc.perform(MockMvcRequestBuilders.fileUpload("/transform")
mockMvc.perform(MockMvcRequestBuilders.multipart("/transform")
.file(sourceFile)
.param("targetExtension", targetExtension)
.param("cropGravity", "badValue"))
@@ -92,7 +207,7 @@ public class ImageMagickControllerTest extends AbstractTransformerControllerTest
{
expectedOptions = "-alpha remove -gravity SouthEast -crop 123x456%+90+12 +repage -thumbnail 321x654%!";
expectedSourceSuffix = "[2-3]";
mockMvc.perform(MockMvcRequestBuilders.fileUpload("/transform")
mockMvc.perform(MockMvcRequestBuilders.multipart("/transform")
.file(sourceFile)
.param("targetExtension", targetExtension)
@@ -126,7 +241,7 @@ public class ImageMagickControllerTest extends AbstractTransformerControllerTest
{
expectedOptions = "-auto-orient -gravity SouthEast -crop 123x456+90+12 +repage -resize 321x654>";
expectedSourceSuffix = "[2-3]";
mockMvc.perform(MockMvcRequestBuilders.fileUpload("/transform")
mockMvc.perform(MockMvcRequestBuilders.multipart("/transform")
.file(sourceFile)
.param("targetExtension", targetExtension)
@@ -160,7 +275,7 @@ public class ImageMagickControllerTest extends AbstractTransformerControllerTest
{
// Example of why the commandOptions parameter is a bad idea.
expectedOptions = "( horrible command / ); -resize 321x654>";
mockMvc.perform(MockMvcRequestBuilders.fileUpload("/transform")
mockMvc.perform(MockMvcRequestBuilders.multipart("/transform")
.file(sourceFile)
.param("targetExtension", targetExtension)
.param("thumbnail", "false")
@@ -180,4 +295,64 @@ public class ImageMagickControllerTest extends AbstractTransformerControllerTest
transformRequest.setSourceMediaType(MediaType.IMAGE_PNG_VALUE);
transformRequest.setTargetMediaType(MediaType.IMAGE_PNG_VALUE);
}
@Test
public void badExitCodeTest() throws Exception
{
when(mockExecutionResult.getExitValue()).thenReturn(1);
mockMvc.perform(mockMvcRequest("/transform", sourceFile, "targetExtension", "xxx"))
.andExpect(status().is(400))
.andExpect(status().reason(containsString("Transformer exit code was not 0: \nSTDERR")));
}
@Test
public void testPojoTransform() throws Exception
{
// Files
String sourceFileRef = UUID.randomUUID().toString();
File sourceFile = getTestFile("quick." + sourceExtension, true);
String targetFileRef = UUID.randomUUID().toString();
// Transformation Request POJO
TransformRequest transformRequest = new TransformRequest();
transformRequest.setRequestId("1");
transformRequest.setSchema(1);
transformRequest.setClientData("Alfresco Digital Business Platform");
transformRequest.setTransformRequestOptions(new HashMap<>());
transformRequest.setSourceReference(sourceFileRef);
transformRequest.setSourceExtension(sourceExtension);
transformRequest.setSourceSize(sourceFile.length());
transformRequest.setTargetExtension(targetExtension);
// HTTP Request
HttpHeaders headers = new HttpHeaders();
headers.set(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=quick." + sourceExtension);
ResponseEntity<Resource> response = new ResponseEntity<>(new FileSystemResource(
sourceFile), headers, HttpStatus.OK);
when(alfrescoSharedFileStoreClient.retrieveFile(sourceFileRef)).thenReturn(response);
when(alfrescoSharedFileStoreClient.saveFile(any())).thenReturn(new FileRefResponse(new FileRefEntity(targetFileRef)));
when(mockExecutionResult.getExitValue()).thenReturn(0);
// Update the Transformation Request with any specific params before sending it
updateTransformRequestWithSpecificOptions(transformRequest);
// Serialize and call the transformer
String tr = objectMapper.writeValueAsString(transformRequest);
String transformationReplyAsString = mockMvc.perform(MockMvcRequestBuilders.post("/transform")
.header(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE)
.header(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE).content(tr))
.andExpect(status().is(HttpStatus.CREATED.value()))
.andReturn().getResponse().getContentAsString();
TransformReply transformReply = objectMapper.readValue(transformationReplyAsString, TransformReply.class);
// Assert the reply
assertEquals(transformRequest.getRequestId(), transformReply.getRequestId());
assertEquals(transformRequest.getClientData(), transformReply.getClientData());
assertEquals(transformRequest.getSchema(), transformReply.getSchema());
}
}

View File

@@ -25,7 +25,6 @@
*/
package org.alfresco.transformer;
import org.alfresco.transformer.AbstractHttpRequestTest;
import org.junit.runner.RunWith;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.context.SpringBootTest.WebEnvironment;
@@ -48,5 +47,5 @@ public class ImageMagickHttpRequestTest extends AbstractHttpRequestTest
protected String getSourceExtension()
{
return "jpg";
};
}
}