ATS-763: Update T-Core for Legacy: Add test files & tests for newly added transforms (in ATS-731) (#252)

* ATS-763: Added missing tests in Ticka

* ATS-763: Added the missing transform tests for Libre Office and replaced quick files in Ticka

* ATS-763: Replaced newly added quick.xml and quick.msg with preexisting files.

* ATS-763: Added targets to tests in Libre Office -see Jan's comment in PR

* ATS-763: Added test files to Image Magick, and uncommented the PSD source file

* ATS-763: put back a comment in Image Magick how it was before my previous commit

* ATS-763: Resolved Jan's comment about seperating out mimetypes into their correct section such as SPREADSHEET or PRESENTATION

* ATS-763: Fixed failing test (ppsm and ppsx)

* ATS-763: Removed unnecessary source files in Image Magick

* ATS-763: Fix failing LibreOffice unit tests

* ATS-763: Fix indentation in LibreOfficeTransformationIT

* ATS-763: fixed failing image magick tests and removed failing transform from config

* ATS-763: Added missing priority for pages -> txt

Co-authored-by: kristian <kristian.dimitrov@alfresco.com>

alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/test/java/org/alfresco/transformer/ImageMagickTransformationIT.java

@@ -116,7 +116,6 @@ public class ImageMagickTransformationIT
         .add(Pair.of("pnm",MIMETYPE_IMAGE_PNM))
         .add(Pair.of("ppj",MIMETYPE_IMAGE_PPJ))
         .add(Pair.of("ppm",MIMETYPE_IMAGE_PPM))
-        .add(Pair.of("psd",MIMETYPE_IMAGE_PSD))
         .add(Pair.of("r3d",MIMETYPE_IMAGE_RAW_R3D))
         .add(Pair.of("raf",MIMETYPE_IMAGE_RAW_RAF))
         .add(Pair.of("ras",MIMETYPE_IMAGE_RAS))
@@ -129,6 +128,33 @@ public class ImageMagickTransformationIT
         .add(Pair.of("xwd",MIMETYPE_IMAGE_XWD))
         .build();
 
+    private static final List<Pair<String,String>> targetExtensionsForPSD = new ImmutableList.Builder<Pair<String,String>>()
+            .add(Pair.of("x3f",MIMETYPE_IMAGE_RAW_X3F))
+            .add(Pair.of("tiff",MIMETYPE_IMAGE_TIFF))
+            .add(Pair.of("rwl",MIMETYPE_IMAGE_RAW_RWL))
+            .add(Pair.of("rw2",MIMETYPE_IMAGE_RAW_RW2))
+            .add(Pair.of("ras",MIMETYPE_IMAGE_RAS))
+            .add(Pair.of("raf",MIMETYPE_IMAGE_RAW_RAF))
+            .add(Pair.of("r3d",MIMETYPE_IMAGE_RAW_R3D))
+            .add(Pair.of("psd",MIMETYPE_IMAGE_PSD))
+            .add(Pair.of("ppm",MIMETYPE_IMAGE_PPM))
+            .add(Pair.of("ppj",MIMETYPE_IMAGE_PPJ))
+            .add(Pair.of("pnm",MIMETYPE_IMAGE_PNM))
+            .add(Pair.of("pgm",MIMETYPE_IMAGE_PGM))
+            .add(Pair.of("pef",MIMETYPE_IMAGE_RAW_PEF))
+            .add(Pair.of("pbm",MIMETYPE_IMAGE_PBM))
+            .add(Pair.of("orf",MIMETYPE_IMAGE_RAW_ORF))
+            .add(Pair.of("nef",MIMETYPE_IMAGE_RAW_NEF))
+            .add(Pair.of("mrw",MIMETYPE_IMAGE_RAW_MRW))
+            .add(Pair.of("k25",MIMETYPE_IMAGE_RAW_K25))
+            .add(Pair.of("ief",MIMETYPE_IMAGE_IEF))
+            .add(Pair.of("gif",MIMETYPE_IMAGE_GIF))
+            .add(Pair.of("dng",MIMETYPE_IMAGE_RAW_DNG))
+            .add(Pair.of("cr2",MIMETYPE_IMAGE_RAW_CR2))
+            .add(Pair.of("arw",MIMETYPE_IMAGE_RAW_ARW))
+            .add(Pair.of("3fr",MIMETYPE_IMAGE_RAW_3FR))
+            .build();
+
     private static final Map<String, TestFileInfo> TEST_FILES = Stream.of(
         testFile(MIMETYPE_IMAGE_BMP,"bmp","quick.bmp"),
         testFile(MIMETYPE_APPLICATION_EPS,"eps","quick.eps"),
@@ -141,9 +167,10 @@ public class ImageMagickTransformationIT
         testFile(MIMETYPE_IMAGE_PPM,"ppm","quick.ppm"),
         testFile(MIMETYPE_IMAGE_XBM,"xbm","quick.xbm"),
         testFile(MIMETYPE_IMAGE_XPM,"xpm","quick.xpm"),
+        testFile(MIMETYPE_IMAGE_PSD,"psd","quick.psd"),
         testFile(MIMETYPE_IMAGE_XWD,"xwd","quick.xwd")
     ).collect(toMap(TestFileInfo::getPath, identity()));
-   
+
     private final String sourceFile;
     private final String targetExtension;
     private final String sourceMimetype;
@@ -160,25 +187,27 @@ public class ImageMagickTransformationIT
     @Parameterized.Parameters
     public static Set<Pair<TestFileInfo, Pair<String,String>>> engineTransformations()
     {
-        return Stream
+        Set<Pair<TestFileInfo, Pair<String,String>>> resolved = null;
+        resolved = Stream
             .of(
-                allTargets("quick.bmp"),
+                allTargets("quick.bmp", targetExtensions),
-                allTargets("quick.eps"),
+                allTargets("quick.eps", targetExtensions),
-                allTargets("quick.gif"),
+                allTargets("quick.gif", targetExtensions),
-                allTargets("quick.jpg"),
+                allTargets("quick.jpg", targetExtensions),
-                allTargets("quick.pbm"),
+                allTargets("quick.pbm", targetExtensions),
-                allTargets("quick.pgm"),
+                allTargets("quick.pgm", targetExtensions),
-                allTargets("quick.png"),
+                allTargets("quick.png", targetExtensions),
-                allTargets("quick.pnm"),
+                allTargets("quick.pnm", targetExtensions),
-                allTargets("quick.ppm"),
+                allTargets("quick.ppm", targetExtensions),
-                //allTargets("quick.psd"),
+                allTargets("quick.psd", targetExtensionsForPSD),
                 //allTargets("quick.tiff"),
-                allTargets("quick.xbm"),
+                allTargets("quick.xbm", targetExtensions),
-                allTargets("quick.xpm"),
+                allTargets("quick.xpm", targetExtensions),
-                allTargets("quick.xwd")
+                allTargets("quick.xwd", targetExtensions)
             )
             .flatMap(identity())
             .collect(toSet());
+        return resolved;
     }
 
     @Test
@@ -198,10 +227,11 @@ public class ImageMagickTransformationIT
         }
     }
 
-    private static Stream<Pair<TestFileInfo, Pair<String,String>>> allTargets(final String sourceFile)
+    private static Stream<Pair<TestFileInfo, Pair<String,String>>> allTargets(final String sourceFile, List<Pair<String,String>> targetExtensionsList)
     {
-        return targetExtensions
+        return targetExtensionsList
             .stream()
             .map(k -> Pair.of(TEST_FILES.get(sourceFile), k));
     }
+
 }

alfresco-transform-imagemagick/alfresco-transform-imagemagick/src/main/resources/imagemagick_engine_config.json

@@ -785,7 +785,6 @@
         {"sourceMediaType": "mage/vnd.adobe.photoshop",                 "targetMediaType": "application/eps" },
         {"sourceMediaType": "mage/vnd.adobe.photoshop",                 "targetMediaType": "image/gif" },
         {"sourceMediaType": "mage/vnd.adobe.photoshop",                 "targetMediaType": "image/ief" },
-        {"sourceMediaType": "mage/vnd.adobe.photoshop",                 "targetMediaType": "image/jp2" },
         {"sourceMediaType": "mage/vnd.adobe.photoshop",                 "targetMediaType": "image/jpeg" },
         {"sourceMediaType": "mage/vnd.adobe.photoshop",                 "targetMediaType": "image/x-raw-kodak" },
         {"sourceMediaType": "mage/vnd.adobe.photoshop",                 "targetMediaType": "image/x-raw-minolta" },

alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/test/java/org/alfresco/transformer/LibreOfficeTransformationIT.java

@@ -51,6 +51,13 @@ import static org.alfresco.transformer.util.MimetypeMap.MIMETYPE_VISIO;
 import static org.alfresco.transformer.util.MimetypeMap.MIMETYPE_VISIO_2013;
 import static org.alfresco.transformer.util.MimetypeMap.MIMETYPE_WORD;
 import static org.alfresco.transformer.util.MimetypeMap.MIMETYPE_WORDPERFECT;
+import static org.alfresco.transformer.util.MimetypeMap.MIMETYPE_XML;
+import static org.alfresco.transformer.util.MimetypeMap.MIMETYPE_OPENXML_SPREADSHEET_TEMPLATE_MACRO;
+import static org.alfresco.transformer.util.MimetypeMap.MIMETYPE_OPENXML_SPREADSHEET_ADDIN_MACRO;
+import static org.alfresco.transformer.util.MimetypeMap.MIMETYPE_OPENXML_PRESENTATION_SLIDESHOW;
+import static org.alfresco.transformer.util.MimetypeMap.MIMETYPE_OPENXML_PRESENTATION_SLIDESHOW_MACRO;
+import static org.alfresco.transformer.util.MimetypeMap.MIMETYPE_OUTLOOK_MSG;
+import static org.alfresco.transformer.util.MimetypeMap.MIMETYPE_DITA;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 import static org.springframework.http.HttpStatus.OK;
@@ -107,29 +114,39 @@ public class LibreOfficeTransformationIT
         testFile(MIMETYPE_PDF,"pdf",null)
     );
 
+    private static final Set<TestFileInfo> pdfTarget = ImmutableSet.of(
+        testFile(MIMETYPE_PDF,"pdf",null)
+    );
+
     private final String sourceFile;
     private final String targetExtension;
     private final String sourceMimetype;
     private final String targetMimetype;
 
     private static final Map<String,TestFileInfo> TEST_FILES = Stream.of(
-        testFile(MIMETYPE_WORD                      ,"doc"  ,"quick.doc"), 
+        testFile(MIMETYPE_WORD                                 ,"doc"  ,"quick.doc"),
-        testFile(MIMETYPE_OPENXML_WORDPROCESSING    ,"docx" ,"quick.docx"),
+        testFile(MIMETYPE_OPENXML_WORDPROCESSING               ,"docx" ,"quick.docx"),
-        testFile(MIMETYPE_OPENDOCUMENT_GRAPHICS     ,"odg"  ,"quick.odg"), 
+        testFile(MIMETYPE_OPENDOCUMENT_GRAPHICS                ,"odg"  ,"quick.odg"),
-        testFile(MIMETYPE_OPENDOCUMENT_PRESENTATION ,"odp"  ,"quick.odp"), 
+        testFile(MIMETYPE_OPENDOCUMENT_PRESENTATION            ,"odp"  ,"quick.odp"),
-        testFile(MIMETYPE_OPENDOCUMENT_SPREADSHEET  ,"ods"  ,"quick.ods"), 
+        testFile(MIMETYPE_OPENDOCUMENT_SPREADSHEET             ,"ods"  ,"quick.ods"),
-        testFile(MIMETYPE_OPENDOCUMENT_TEXT         ,"odt"  ,"quick.odt"), 
+        testFile(MIMETYPE_OPENDOCUMENT_TEXT                    ,"odt"  ,"quick.odt"),
-        testFile(MIMETYPE_PPT                       ,"ppt"  ,"quick.ppt"), 
+        testFile(MIMETYPE_PPT                                  ,"ppt"  ,"quick.ppt"),
-        testFile(MIMETYPE_OPENXML_PRESENTATION      ,"pptx" ,"quick.pptx"),
+        testFile(MIMETYPE_OPENXML_PRESENTATION                 ,"pptx" ,"quick.pptx"),
-        testFile(MIMETYPE_VISIO                     ,"vdx"  ,"quick.vdx"), 
+        testFile(MIMETYPE_VISIO                                ,"vdx"  ,"quick.vdx"),
-        testFile(MIMETYPE_VISIO_2013                ,"vsd"  ,"quick.vsd"), 
+        testFile(MIMETYPE_VISIO_2013                           ,"vsd"  ,"quick.vsd"),
-        testFile(MIMETYPE_WORDPERFECT               ,"wpd"  ,"quick.wpd"), 
+        testFile(MIMETYPE_WORDPERFECT                          ,"wpd"  ,"quick.wpd"),
-        testFile(MIMETYPE_EXCEL                     ,"xls"  ,"quick.xls" ), 
+        testFile(MIMETYPE_EXCEL                                ,"xls"  ,"quick.xls" ),
-        testFile(MIMETYPE_OPENXML_SPREADSHEET       ,"xlsx" ,"quick.xlsx"),
+        testFile(MIMETYPE_OPENXML_SPREADSHEET                  ,"xlsx" ,"quick.xlsx"),
-        testFile(MIMETYPE_TEXT_CSV                  ,"csv"  ,"people.csv"),
+        testFile(MIMETYPE_TEXT_CSV                             ,"csv"  ,"people.csv"),
-        testFile(MIMETYPE_RTF                       ,"rtf"  ,"sample.rtf"),
+        testFile(MIMETYPE_RTF                                  ,"rtf"  ,"sample.rtf"),
-        testFile(MIMETYPE_HTML                      ,"html" ,"quick.html"),
+        testFile(MIMETYPE_HTML                                 ,"html" ,"quick.html"),
-        testFile(MIMETYPE_TSV                       ,"tsv"  ,"sample.tsv")
+        testFile(MIMETYPE_XML                                  ,"xml" ,"quick.xml"),
+        testFile(MIMETYPE_OPENXML_SPREADSHEET_TEMPLATE_MACRO   ,"xltm" ,"quick.xltm"),
+        testFile(MIMETYPE_OPENXML_PRESENTATION_SLIDESHOW       ,"ppsx" ,"quick.ppsx"),
+        testFile(MIMETYPE_OPENXML_PRESENTATION_SLIDESHOW_MACRO ,"ppsm" ,"quick.ppsm"),
+        testFile(MIMETYPE_OUTLOOK_MSG                          ,"msg" ,"quick.msg"),
+        testFile(MIMETYPE_DITA                                 ,"dita" ,"quick.dita"),
+        testFile(MIMETYPE_TSV                                  ,"tsv"  ,"sample.tsv")
     ).collect(toMap(TestFileInfo::getPath, identity()));
 
     public LibreOfficeTransformationIT(final Pair<TestFileInfo, TestFileInfo> entry)
@@ -159,13 +176,20 @@ public class LibreOfficeTransformationIT
                 allTargets("quick.odg", graphicTargets),
                 allTargets("quick.vdx", graphicTargets),
                 allTargets("quick.vsd", graphicTargets),
-                
+
                 allTargets("quick.ods", spreadsheetTargets),
                 allTargets("quick.xls", spreadsheetTargets),
                 allTargets("quick.xlsx", spreadsheetTargets),
                 allTargets("people.csv", spreadsheetTargets),
-                allTargets("sample.tsv", spreadsheetTargets)
+                allTargets("sample.tsv", spreadsheetTargets),
-            )
+
+                allTargets("quick.xml", pdfTarget),
+                allTargets("quick.xltm", pdfTarget),
+                allTargets("quick.dita", pdfTarget),
+                allTargets("quick.msg", pdfTarget),
+                allTargets("quick.ppsm", pdfTarget),
+                allTargets("quick.ppsx", pdfTarget)
+                )
             .flatMap(identity())
             .collect(toSet());
     }
@@ -193,7 +217,7 @@ public class LibreOfficeTransformationIT
         return mimetypes
             .stream()
             //Filter out duplicate mimetypes. eg. We do not want "Transform (quick.doc, application/msword -> application/msword, doc)" as these are not contained in the engine_config
-            .filter(type -> !type.getMimeType().equals(TEST_FILES.get(sourceFile).getMimeType())) 
+            .filter(type -> !type.getMimeType().equals(TEST_FILES.get(sourceFile).getMimeType()))
             // Edge case: Transform (quick.ods, application/vnd.oasis.opendocument.spreadsheet -> text/csv, csv) not in engine_config
             .filter(type -> !(TEST_FILES.get(sourceFile).getMimeType().equals(MIMETYPE_OPENDOCUMENT_SPREADSHEET) && type.getMimeType().equals(MIMETYPE_TEXT_CSV)))
             .map(k -> Pair.of(TEST_FILES.get(sourceFile), k));

alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/test/resources/quick.dita

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE concept PUBLIC "-//OASIS//DTD DITA Concept//EN" "http://docs.oasis-open.org/dita/v1.1/OS/dtd/concept.dtd">
+<concept id="quickConcept">
+ <title>The quick brown fox jumps over the lazy dog</title>
+ <shortdesc>Gym class featuring a brown fox and lazy dog</shortdesc>
+ <prolog>
+  <author>Alfresco Documentation</author>
+  <copyright>
+   <copyryear year="2011"/>
+   <copyrholder>Alfresco Software, Inc.</copyrholder>
+  </copyright>
+  <metadata>
+   <audience experiencelevel="expert" job="Customizing" type="Administrator"/>
+   <category>Testing</category>
+   <keywords>
+    <keyword>Pangram</keyword>
+    <keyword>Fox</keyword>
+    <keyword>Dog</keyword>
+   </keywords>
+   <prodinfo>
+    <prodname>Enterprise</prodname>
+    <vrmlist>
+     <vrm version="3.4.x" release="Enterprise" modification="2011/11/11"/>
+    </vrmlist>
+   </prodinfo>
+  </metadata>
+ </prolog>
+ <conbody>
+   <p>The quick brown fox jumps over the lazy dog</p>
+ </conbody>
+</concept>

alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/test/resources/quick.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+
+<document>
+   <text>The quick brown fox jumps over the lazy dog</text>
+</document>

alfresco-transform-libreoffice/alfresco-transform-libreoffice/src/main/resources/libreoffice_engine_config.json

@@ -227,8 +227,6 @@
 
         {"sourceMediaType": "application/vnd.ms-excel.template.macroenabled.12",                                                        "priority": 110, "targetMediaType": "application/pdf" },
 
-        {"sourceMediaType": "application/vnd.ms-excel.addin.macroenabled.12",                                                           "priority": 110, "targetMediaType": "application/pdf" },
-
         {"sourceMediaType": "application/vnd.openxmlformats-officedocument.presentationml.slideshow",                                   "priority": 110, "targetMediaType": "application/pdf" },
 
         {"sourceMediaType": "application/vnd.ms-powerpoint.slideshow.macroenabled.12",                                                  "priority": 110, "targetMediaType": "application/pdf" },

alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transformer/TikaTransformationIT.java

@@ -138,6 +138,8 @@ public class TikaTransformationIT
                 allTargets("sample.xhtml.txt", "application/xhtml+xml"),
                 allTargets("sample.rss", "application/rss+xml"),
                 //allTargets("quick.rar", "application/x-rar-compressed"),
+                allTargets("quick.z", "application/x-compress"),
+                allTargets("quick.csv", "text/csv"),
                 allTargets("quick.tar.gz", "application/x-gzip"))
             .flatMap(identity())
             .collect(toSet());

alfresco-transform-tika/alfresco-transform-tika-boot/src/test/resources/quick.csv

@@ -0,0 +1 @@
+"The quick brown fox jumps over the lazy dog"

alfresco-transform-tika/alfresco-transform-tika-boot/src/test/resources/quick.z

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?><html xmlns="http://www.w3.org/1999/xhtml">
+    
+    <head>
+        
+        <meta name="X-Parsed-By" content="org.apache.tika.parser.DefaultParser"/>
+        
+        <meta name="X-Parsed-By" content="org.apache.tika.parser.pkg.CompressorParser"/>
+        
+        <meta name="Content-Type" content="application/gzip"/>
+        
+        <title/>
+        
+    </head>
+    
+    <body>
+
+        The quick brown fox jumps over the lazy dog
+
+        <div class="package-entry">
+        </div>
+        
+    </body>
+</html>

alfresco-transform-tika/alfresco-transform-tika/src/main/resources/tika_engine_config.json

@@ -354,7 +354,7 @@
         {"sourceMediaType": "application/vnd.apple.pages",                                                                 "targetMediaType": "text/html"},
         {"sourceMediaType": "application/vnd.apple.pages",                                                                 "targetMediaType": "application/xhtml+xml"},
         {"sourceMediaType": "application/vnd.apple.pages",                                                                 "targetMediaType": "text/xml"},
-        {"sourceMediaType": "application/vnd.apple.pages",                                                                 "targetMediaType": "text/plain"},
+        {"sourceMediaType": "application/vnd.apple.pages",                                                "priority": 120, "targetMediaType": "text/plain"},
 
         {"sourceMediaType": "application/pdf",                                                             "priority": 55, "targetMediaType": "text/html"},
         {"sourceMediaType": "application/pdf",                             "maxSourceSizeBytes": 26214400, "priority": 55, "targetMediaType": "text/plain"},