From 3806ef2f25261c7b1760848bfea4e4a0075418f2 Mon Sep 17 00:00:00 2001
From: eknizat <26163420+eknizat@users.noreply.github.com>
Date: Wed, 28 Aug 2019 09:59:54 +0100
Subject: [PATCH 01/20] REPO-4616: Add EMLTransformer to the Misc T-Engine
(#96)
* REPO-4616: Add EMLTransformer to the Misc T-Engine
---
alfresco-docker-transform-misc/pom.xml | 7 +
.../AppleIWorksContentTransformer.java | 4 +
.../transformers/EMLTransformer.java | 245 ++++++++++++++++++
.../HtmlParserContentTransformer.java | 2 +-
.../OOXMLThumbnailContentTransformer.java | 4 +
.../transformers/SelectingTransformer.java | 3 +-
.../src/main/resources/engine_config.json | 8 +
.../transformer/MiscControllerTest.java | 211 +++++++++++----
.../transformer/MiscTransformsIT.java | 7 +-
.../src/test/resources/quick.alternative.eml | 30 +++
.../src/test/resources/quick.attachment.eml | 44 ++++
.../src/test/resources/quick.eml | 10 +
.../src/test/resources/quick.htmlChars.eml | 28 ++
.../resources/quick.nested.alternative.eml | 41 +++
.../src/test/resources/quick.spanish.eml | 31 +++
15 files changed, 616 insertions(+), 59 deletions(-)
create mode 100644 alfresco-docker-transform-misc/src/main/java/org/alfresco/transformer/transformers/EMLTransformer.java
create mode 100644 alfresco-docker-transform-misc/src/test/resources/quick.alternative.eml
create mode 100644 alfresco-docker-transform-misc/src/test/resources/quick.attachment.eml
create mode 100644 alfresco-docker-transform-misc/src/test/resources/quick.eml
create mode 100644 alfresco-docker-transform-misc/src/test/resources/quick.htmlChars.eml
create mode 100644 alfresco-docker-transform-misc/src/test/resources/quick.nested.alternative.eml
create mode 100644 alfresco-docker-transform-misc/src/test/resources/quick.spanish.eml
diff --git a/alfresco-docker-transform-misc/pom.xml b/alfresco-docker-transform-misc/pom.xml
index 6da64342..8bf726e3 100644
--- a/alfresco-docker-transform-misc/pom.xml
+++ b/alfresco-docker-transform-misc/pom.xml
@@ -79,6 +79,13 @@
poi-ooxml
3.17
+
+
+
+ com.sun.mail
+ javax.mail
+ 1.6.2
+
diff --git a/alfresco-docker-transform-misc/src/main/java/org/alfresco/transformer/transformers/AppleIWorksContentTransformer.java b/alfresco-docker-transform-misc/src/main/java/org/alfresco/transformer/transformers/AppleIWorksContentTransformer.java
index 38edf4f5..64d744db 100644
--- a/alfresco-docker-transform-misc/src/main/java/org/alfresco/transformer/transformers/AppleIWorksContentTransformer.java
+++ b/alfresco-docker-transform-misc/src/main/java/org/alfresco/transformer/transformers/AppleIWorksContentTransformer.java
@@ -54,6 +54,10 @@ import com.google.common.collect.ImmutableList;
* assume incorrectly that we can convert to PDF and we would only get a preview for the older format and never the
* newer one. Both formats have the same mimetype.
*
+ *
+ * This code is based on a class of the same name originally implemented in alfresco-repository.
+ *
+ *
* @author Neil Mc Erlean
* @author eknizat
* @since 4.0
diff --git a/alfresco-docker-transform-misc/src/main/java/org/alfresco/transformer/transformers/EMLTransformer.java b/alfresco-docker-transform-misc/src/main/java/org/alfresco/transformer/transformers/EMLTransformer.java
new file mode 100644
index 00000000..0710635e
--- /dev/null
+++ b/alfresco-docker-transform-misc/src/main/java/org/alfresco/transformer/transformers/EMLTransformer.java
@@ -0,0 +1,245 @@
+/*
+ * #%L
+ * Alfresco Transform Core
+ * %%
+ * Copyright (C) 2005 - 2019 Alfresco Software Limited
+ * %%
+ * This file is part of the Alfresco software.
+ * -
+ * If the software was purchased under a paid Alfresco license, the terms of
+ * the paid license agreement will prevail. Otherwise, the software is
+ * provided under the following open source license terms:
+ * -
+ * Alfresco is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ * -
+ * Alfresco is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ * -
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with Alfresco. If not, see .
+ * #L%
+ */
+package org.alfresco.transformer.transformers;
+
+import org.alfresco.transformer.fs.FileManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.alfresco.transform.client.model.Mimetype.MIMETYPE_HTML;
+import static org.alfresco.transform.client.model.Mimetype.MIMETYPE_MULTIPART_ALTERNATIVE;
+import static org.alfresco.transform.client.model.Mimetype.MIMETYPE_RFC822;
+import static org.alfresco.transform.client.model.Mimetype.MIMETYPE_TEXT_PLAIN;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedWriter;
+import java.io.File;
+
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.util.Map;
+import java.util.Properties;
+
+import javax.mail.MessagingException;
+import javax.mail.Multipart;
+import javax.mail.Part;
+import javax.mail.Session;
+import javax.mail.internet.MimeMessage;
+
+
+/**
+ * Uses javax.mail.MimeMessage to generate plain text versions of RFC822 email
+ * messages. Searches for all text content parts, and returns them. Any
+ * attachments are ignored. TIKA Note - could be replaced with the Tika email
+ * parser. Would require a recursing parser to be specified, but not the full
+ * Auto one (we don't want attachments), just one containing text and html
+ * related parsers.
+ *
+ *
+ * This code is based on a class of the same name originally implemented in alfresco-repository.
+ *
+ *
+ */
+public class EMLTransformer implements SelectableTransformer
+
+{
+ private static final Logger logger = LoggerFactory.getLogger( EMLTransformer.class);
+
+ private static final String CHARSET = "charset";
+ private static final String DEFAULT_ENCODING = "UTF-8";
+
+ @Override
+ public boolean isTransformable(String sourceMimetype, String targetMimetype, Map parameters)
+ {
+ if (!MIMETYPE_RFC822.equals(sourceMimetype)
+ || !MIMETYPE_TEXT_PLAIN.equals(targetMimetype))
+ {
+ // only support RFC822 -> TEXT
+ return false;
+ }
+ else
+ {
+ return true;
+ }
+ }
+
+ @Override
+ public void transform(final File sourceFile, final File targetFile, final String sourceMimetype,
+ final String targetMimetype, final Map parameters) throws Exception
+ {
+ logger.debug("Performing RFC822 to text transform.");
+ // Use try with resource
+ try (InputStream contentInputStream = new BufferedInputStream( new FileInputStream(sourceFile));
+ Writer bufferedFileWriter = new BufferedWriter(new FileWriter(targetFile)))
+ {
+ MimeMessage mimeMessage = new MimeMessage(Session.getDefaultInstance(new Properties()), contentInputStream);
+
+ final StringBuilder sb = new StringBuilder();
+ Object content = mimeMessage.getContent();
+ if (content instanceof Multipart)
+ {
+ processMultiPart((Multipart) content,sb);
+ }
+ else
+ {
+ sb.append(content.toString());
+ }
+ bufferedFileWriter.write(sb.toString());
+ }
+ }
+
+ /**
+ * Find "text" parts of message recursively and appends it to sb StringBuilder
+ *
+ * @param multipart Multipart to process
+ * @param sb StringBuilder
+ * @throws MessagingException
+ * @throws IOException
+ */
+ private void processMultiPart(Multipart multipart, StringBuilder sb) throws MessagingException, IOException
+ {
+ boolean isAlternativeMultipart = multipart.getContentType().contains(MIMETYPE_MULTIPART_ALTERNATIVE);
+ if (isAlternativeMultipart)
+ {
+ processAlternativeMultipart(multipart, sb);
+ }
+ else
+ {
+ for (int i = 0, n = multipart.getCount(); i < n; i++)
+ {
+ Part part = multipart.getBodyPart(i);
+ if (part.getContent() instanceof Multipart)
+ {
+ processMultiPart((Multipart) part.getContent(), sb);
+ }
+ else
+ {
+ processPart(part, sb);
+ }
+ }
+ }
+ }
+
+ /**
+ * Finds the suitable part from an multipart/alternative and appends it's text content to StringBuilder sb
+ *
+ * @param multipart
+ * @param sb
+ * @throws IOException
+ * @throws MessagingException
+ */
+ private void processAlternativeMultipart(Multipart multipart, StringBuilder sb) throws IOException, MessagingException
+ {
+ Part partToUse = null;
+ for (int i = 0, n = multipart.getCount(); i < n; i++)
+ {
+ Part part = multipart.getBodyPart(i);
+ if (part.getContentType().contains(MIMETYPE_TEXT_PLAIN))
+ {
+ partToUse = part;
+ break;
+ }
+ else if (part.getContentType().contains(MIMETYPE_HTML))
+ {
+ partToUse = part;
+ }
+ else if (part.getContentType().contains(MIMETYPE_MULTIPART_ALTERNATIVE))
+ {
+ if (part.getContent() instanceof Multipart)
+ {
+ processAlternativeMultipart((Multipart) part.getContent(), sb);
+ }
+ }
+ }
+ if (partToUse != null)
+ {
+ processPart(partToUse, sb);
+ }
+ }
+
+ /**
+ * Finds text on a given mail part. Accepted parts types are text/html and text/plain.
+ * Attachments are ignored
+ *
+ * @param part
+ * @param sb
+ * @throws IOException
+ * @throws MessagingException
+ */
+ private void processPart(Part part, StringBuilder sb) throws IOException, MessagingException
+ {
+ boolean isAttachment = Part.ATTACHMENT.equalsIgnoreCase(part.getDisposition());
+ if (isAttachment)
+ {
+ return;
+ }
+ if (part.getContentType().contains(MIMETYPE_TEXT_PLAIN))
+ {
+ sb.append(part.getContent().toString());
+ }
+ else if (part.getContentType().contains(MIMETYPE_HTML))
+ {
+ String mailPartContent = part.getContent().toString();
+
+ //create a temporary html file with same mail part content and encoding
+ File tempHtmlFile = FileManager.TempFileProvider.createTempFile("EMLTransformer_", ".html");
+ String encoding = getMailPartContentEncoding(part);
+ try (OutputStreamWriter osWriter = new OutputStreamWriter( new FileOutputStream( tempHtmlFile), encoding))
+ {
+ osWriter.write(mailPartContent);
+ }
+
+ //transform html file's content to plain text
+ HtmlParserContentTransformer.EncodingAwareStringBean extractor = new HtmlParserContentTransformer.EncodingAwareStringBean();
+ extractor.setCollapse(false);
+ extractor.setLinks(false);
+ extractor.setReplaceNonBreakingSpaces(false);
+ extractor.setURL(tempHtmlFile, encoding);
+ sb.append(extractor.getStrings());
+
+ tempHtmlFile.delete();
+ }
+ }
+
+ private String getMailPartContentEncoding(Part part) throws MessagingException
+ {
+ String encoding = DEFAULT_ENCODING;
+ String contentType = part.getContentType();
+ int startIndex = contentType.indexOf(CHARSET);
+ if (startIndex > 0)
+ {
+ encoding = contentType.substring(startIndex + CHARSET.length() + 1).replaceAll("\"", "");
+ }
+ return encoding;
+ }
+
+}
\ No newline at end of file
diff --git a/alfresco-docker-transform-misc/src/main/java/org/alfresco/transformer/transformers/HtmlParserContentTransformer.java b/alfresco-docker-transform-misc/src/main/java/org/alfresco/transformer/transformers/HtmlParserContentTransformer.java
index 033aa092..94e87f7e 100644
--- a/alfresco-docker-transform-misc/src/main/java/org/alfresco/transformer/transformers/HtmlParserContentTransformer.java
+++ b/alfresco-docker-transform-misc/src/main/java/org/alfresco/transformer/transformers/HtmlParserContentTransformer.java
@@ -143,7 +143,7 @@ public class HtmlParserContentTransformer implements SelectableTransformer
* is specified against the content property (rather than in the
* HTML Head Meta), see ALF-10466 for details.
*/
- private class EncodingAwareStringBean extends StringBean
+ public static class EncodingAwareStringBean extends StringBean
{
private static final long serialVersionUID = -9033414360428669553L;
diff --git a/alfresco-docker-transform-misc/src/main/java/org/alfresco/transformer/transformers/OOXMLThumbnailContentTransformer.java b/alfresco-docker-transform-misc/src/main/java/org/alfresco/transformer/transformers/OOXMLThumbnailContentTransformer.java
index d32830a6..4884ffbd 100644
--- a/alfresco-docker-transform-misc/src/main/java/org/alfresco/transformer/transformers/OOXMLThumbnailContentTransformer.java
+++ b/alfresco-docker-transform-misc/src/main/java/org/alfresco/transformer/transformers/OOXMLThumbnailContentTransformer.java
@@ -70,6 +70,10 @@ import com.google.common.collect.ImmutableList;
* This transformer will only work for OOXML files where thumbnailing was enabled,
* which isn't on by default on Windows, but is more common on Mac.
*
+ *
+ * This code is based on a class of the same name originally implemented in alfresco-repository.
+ *
+ *
* @author Nick Burch
* @author eknizat
*/
diff --git a/alfresco-docker-transform-misc/src/main/java/org/alfresco/transformer/transformers/SelectingTransformer.java b/alfresco-docker-transform-misc/src/main/java/org/alfresco/transformer/transformers/SelectingTransformer.java
index 4abd8629..0e03f375 100644
--- a/alfresco-docker-transform-misc/src/main/java/org/alfresco/transformer/transformers/SelectingTransformer.java
+++ b/alfresco-docker-transform-misc/src/main/java/org/alfresco/transformer/transformers/SelectingTransformer.java
@@ -61,7 +61,8 @@ public class SelectingTransformer
new AppleIWorksContentTransformer(),
new HtmlParserContentTransformer(),
new StringExtractingContentTransformer(),
- new TextToPdfContentTransformer()
+ new TextToPdfContentTransformer(),
+ new EMLTransformer()
// new OOXMLThumbnailContentTransformer()); // Doesn't work with java 11, transformer and test disabled
);
}
diff --git a/alfresco-docker-transform-misc/src/main/resources/engine_config.json b/alfresco-docker-transform-misc/src/main/resources/engine_config.json
index a12997d2..f2b7a485 100644
--- a/alfresco-docker-transform-misc/src/main/resources/engine_config.json
+++ b/alfresco-docker-transform-misc/src/main/resources/engine_config.json
@@ -60,6 +60,14 @@
"transformOptions": [
"textToPdfOptions"
]
+ },
+ {
+ "transformerName": "rfc822",
+ "supportedSourceAndTargetList": [
+ {"sourceMediaType": "message/rfc822", "targetMediaType": "text/plain"}
+ ],
+ "transformOptions": [
+ ]
}
]
}
\ No newline at end of file
diff --git a/alfresco-docker-transform-misc/src/test/java/org/alfresco/transformer/MiscControllerTest.java b/alfresco-docker-transform-misc/src/test/java/org/alfresco/transformer/MiscControllerTest.java
index cbb8a19c..7830a804 100644
--- a/alfresco-docker-transform-misc/src/test/java/org/alfresco/transformer/MiscControllerTest.java
+++ b/alfresco-docker-transform-misc/src/test/java/org/alfresco/transformer/MiscControllerTest.java
@@ -31,11 +31,12 @@ import static org.alfresco.transform.client.model.Mimetype.MIMETYPE_HTML;
import static org.alfresco.transform.client.model.Mimetype.MIMETYPE_IMAGE_JPEG;
import static org.alfresco.transform.client.model.Mimetype.MIMETYPE_IWORK_KEYNOTE;
import static org.alfresco.transform.client.model.Mimetype.MIMETYPE_IWORK_NUMBERS;
-import static org.alfresco.transform.client.model.Mimetype.MIMETYPE_IWORK_PAGES;
import static org.alfresco.transform.client.model.Mimetype.MIMETYPE_OPENXML_WORDPROCESSING;
import static org.alfresco.transform.client.model.Mimetype.MIMETYPE_PDF;
import static org.alfresco.transform.client.model.Mimetype.MIMETYPE_TEXT_PLAIN;
+import static org.alfresco.transform.client.model.Mimetype.MIMETYPE_RFC822;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.springframework.http.HttpStatus.OK;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header;
@@ -105,7 +106,6 @@ public class MiscControllerTest extends AbstractTransformerControllerTest
}
@Override
- // Add extra required parameters to the request.
protected MockHttpServletRequestBuilder mockMvcRequest(String url, MockMultipartFile sourceFile,
String... params)
{
@@ -124,6 +124,116 @@ public class MiscControllerTest extends AbstractTransformerControllerTest
// It is the mock that returns a zero length file for other transformers, when we supply an invalid targetExtension.
}
+ /**
+ * Test transforming a valid eml file to text
+ */
+ @Test
+ public void testRFC822ToText() throws Exception
+ {
+ String expected = "Gym class featuring a brown fox and lazy dog";
+ MvcResult result = sendRequest("eml",
+ null,
+ MIMETYPE_RFC822,
+ "txt",
+ MIMETYPE_TEXT_PLAIN,
+ null,
+ readTestFile("eml"));
+ assertTrue("Content from eml transform didn't contain expected value. ",
+ result.getResponse().getContentAsString().contains(expected));
+ }
+
+ /**
+ * Test transforming a non-ascii eml file to text
+ */
+ @Test
+ public void testNonAsciiRFC822ToText() throws Exception
+ {
+ String expected = "El r\u00E1pido zorro marr\u00F3n salta sobre el perro perezoso";
+ MvcResult result = sendRequest("eml",
+ null,
+ MIMETYPE_RFC822,
+ "txt",
+ MIMETYPE_TEXT_PLAIN,
+ null,
+ readTestFile("spanish.eml"));
+
+ String contentResult = new String(result.getResponse().getContentAsByteArray(), "UTF-8");
+ assertTrue("Content from eml transform didn't contain expected value. ", contentResult.contains(expected));
+ }
+
+ /**
+ * Test transforming a valid eml with an attachment to text; attachment should be ignored
+ */
+ @Test
+ public void testRFC822WithAttachmentToText() throws Exception
+ {
+ String expected = "Mail with attachment content";
+ String notExpected = "File attachment content";
+ MvcResult result = sendRequest("eml",
+ null,
+ MIMETYPE_RFC822,
+ "txt",
+ MIMETYPE_TEXT_PLAIN,
+ null,
+ readTestFile("attachment.eml"));
+ assertTrue("Content from eml transform didn't contain expected value. ",
+ result.getResponse().getContentAsString().contains(expected));
+ assertFalse(result.getResponse().getContentAsString().contains(notExpected));
+ }
+
+ /**
+ * Test transforming a valid eml with minetype multipart/alternative to text
+ */
+ @Test
+ public void testRFC822AlternativeToText() throws Exception
+ {
+ String expected = "alternative plain text";
+ MvcResult result = sendRequest("eml",
+ null,
+ MIMETYPE_RFC822,
+ "txt",
+ MIMETYPE_TEXT_PLAIN,
+ null,
+ readTestFile("alternative.eml"));
+ assertTrue("Content from eml transform didn't contain expected value. ",
+ result.getResponse().getContentAsString().contains(expected));
+ }
+
+ /**
+ * Test transforming a valid eml with nested mimetype multipart/alternative to text
+ */
+ @Test
+ public void testRFC822NestedAlternativeToText() throws Exception
+ {
+ String expected = "nested alternative plain text";
+ MvcResult result = sendRequest("eml",
+ null,
+ MIMETYPE_RFC822,
+ "txt",
+ MIMETYPE_TEXT_PLAIN,
+ null,
+ readTestFile("nested.alternative.eml"));
+ assertTrue("Content from eml transform didn't contain expected value. ",
+ result.getResponse().getContentAsString().contains(expected));
+ }
+
+ /**
+ * Test transforming a valid eml with a html part containing html special characters to text
+ */
+ @Test
+ public void testHtmlSpecialCharsToText() throws Exception
+ {
+ String expected = " ";
+ MvcResult result = sendRequest("eml",
+ null,
+ MIMETYPE_RFC822,
+ "txt",
+ MIMETYPE_TEXT_PLAIN,
+ null,
+ readTestFile("htmlChars.eml"));
+ assertFalse(result.getResponse().getContentAsString().contains(expected));
+ }
+
@Test
public void testHTMLtoString() throws Exception
{
@@ -139,7 +249,7 @@ public class MiscControllerTest extends AbstractTransformerControllerTest
String partC = "