inteligr8/alfresco-transform-core
1
0
Fork 0
mirror of https://github.com/Alfresco/alfresco-transform-core.git synced 2025-07-31 17:38:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix/mnt 25089 html transformations with ootb aio create extra whitespace (#1079)

Browse Source
This commit is contained in:
KushalBanik
2025-06-03 13:23:33 +05:30
committed by GitHub
parent 0c534f1081
commit cb9d070c9c
14 changed files with 1496 additions and 1324 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
engines/aio/src/main/resources/application-default.yaml
View File

@@ -26,4 +26,6 @@ transform:
unixOS: 'env FOO=#{"$"}{OUTPUT} exiftool -args -G1 -sep "|||" #{"$"}{INPUT}'
misc:
pdfBox:
defaultFont: ${MISC_PDFBOX_DEFAULT_FONT:NotoSans-Regular}
defaultFont: ${MISC_PDFBOX_DEFAULT_FONT:NotoSans-Regular}
htmlOptions:
collapseHtml: ${MISC_HTML_COLLAPSE:true}

254
engines/aio/src/test/java/org/alfresco/transform/aio/AIOTest.java
View File

@@ -1,126 +1,128 @@
/*
* #%L
* Alfresco Transform Core
* %%
* Copyright (C) 2005 - 2022 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* -
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
* -
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
* -
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
* -
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
package org.alfresco.transform.aio;
import org.alfresco.transform.base.AbstractBaseTest;
import org.alfresco.transform.base.TransformController;
import org.alfresco.transform.config.TransformConfig;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.ResponseEntity;
import org.springframework.mock.web.MockMultipartFile;
import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder;
import java.nio.file.Files;
import java.util.StringJoiner;
import static org.alfresco.transform.base.TransformControllerTest.getLogMessagesFor;
import static org.alfresco.transform.common.Mimetype.MIMETYPE_HTML;
import static org.alfresco.transform.common.Mimetype.MIMETYPE_TEXT_PLAIN;
import static org.alfresco.transform.common.RequestParamMap.CONFIG_VERSION_DEFAULT;
import static org.alfresco.transform.common.RequestParamMap.CONFIG_VERSION_LATEST;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
/**
* Test All-In-One.
*/
public class AIOTest extends AbstractBaseTest
{
@Autowired
private String coreVersion;
@BeforeEach
public void before() throws Exception
{
sourceMimetype = MIMETYPE_HTML;
targetMimetype = MIMETYPE_TEXT_PLAIN;
sourceExtension = "html";
targetExtension = "txt";
expectedOptions = null;
expectedSourceSuffix = null;
sourceFileBytes = readTestFile(sourceExtension);
expectedTargetFileBytes = Files.readAllBytes(getTestFile("quick2." + targetExtension, true).toPath());
sourceFile = new MockMultipartFile("file", "quick." + sourceExtension, sourceMimetype, sourceFileBytes);
}
@Override
// Add extra required parameters to the request.
protected MockHttpServletRequestBuilder mockMvcRequest(String url, MockMultipartFile sourceFile, String... params)
{
return super.mockMvcRequest(url, sourceFile, params)
.param("targetMimetype", targetMimetype)
.param("sourceMimetype", sourceMimetype);
}
@Test
public void coreVersionNotSetInOriginalConfigTest()
{
ResponseEntity<TransformConfig> responseEntity = controller.transformConfig(Integer.valueOf(CONFIG_VERSION_DEFAULT));
responseEntity.getBody().getTransformers().forEach(transformer -> {
assertNull(transformer.getCoreVersion(), transformer.getTransformerName() +
" should have had a null coreValue but was " + transformer.getCoreVersion());
});
}
@Test
public void coreVersionSetInLatestConfigTest()
{
ResponseEntity<TransformConfig> responseEntity = controller.transformConfig(CONFIG_VERSION_LATEST);
responseEntity.getBody().getTransformers().forEach(transformer -> {
assertNotNull(transformer.getCoreVersion(), transformer.getTransformerName() +
" should have had a coreValue but was null. Should have been " + coreVersion);
});
}
@Test
public void testStartupLogsIncludeEngineMessages()
{
StringJoiner controllerLogMessages = getLogMessagesFor(TransformController.class);
controller.startup();
assertEquals(
"--------------------------------------------------------------------------------------------------------------------------------------------------------------\n"
+ "If the Alfresco software was purchased under a paid Alfresco license, the terms of the paid license agreement \n"
+ "will prevail. Otherwise, the software is provided under terms of the GNU LGPL v3 license. \n"
+ "See the license at http://www.gnu.org/licenses/lgpl-3.0.txt. or in /LICENSE.txt \n"
+ "\n"
+ "This transformer uses ImageMagick from ImageMagick Studio LLC. See the license at http://www.imagemagick.org/script/license.php or in /ImageMagick-license.txt\n"
+ "This transformer uses LibreOffice from The Document Foundation. See the license at https://www.libreoffice.org/download/license/ or in /libreoffice.txt\n"
+ "This transformer uses libraries from Apache. See the license at http://www.apache.org/licenses/LICENSE-2.0. or in /Apache\\\\ 2.0.txt\n"
+ "This transformer uses htmlparser. See the license at http://htmlparser.sourceforge.net/license.html\n"
+ "This transformer uses alfresco-pdf-renderer which uses the PDFium library from Google Inc. See the license at https://pdfium.googlesource.com/pdfium/+/master/LICENSE or in /pdfium.txt\n"
+ "This transformer uses Tika from Apache. See the license at http://www.apache.org/licenses/LICENSE-2.0. or in /Apache\\ 2.0.txt\n"
+ "This transformer uses ExifTool by Phil Harvey. See license at https://exiftool.org/#license. or in /Perl-Artistic-License.txt\n"
+ "--------------------------------------------------------------------------------------------------------------------------------------------------------------\n"
+ "Starting application components... Done",
controllerLogMessages.toString());
}
}
/*
* #%L
* Alfresco Transform Core
* %%
* Copyright (C) 2005 - 2025 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* -
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
* -
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
* -
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
* -
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
package org.alfresco.transform.aio;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.alfresco.transform.base.TransformControllerTest.getLogMessagesFor;
import static org.alfresco.transform.common.Mimetype.MIMETYPE_HTML;
import static org.alfresco.transform.common.Mimetype.MIMETYPE_TEXT_PLAIN;
import static org.alfresco.transform.common.RequestParamMap.*;
import java.nio.file.Files;
import java.util.StringJoiner;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.ResponseEntity;
import org.springframework.mock.web.MockMultipartFile;
import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder;
import org.alfresco.transform.base.AbstractBaseTest;
import org.alfresco.transform.base.TransformController;
import org.alfresco.transform.config.TransformConfig;
/**
* Test All-In-One
*/
public class AIOTest extends AbstractBaseTest
{
@Autowired
private String coreVersion;
@BeforeEach
public void before() throws Exception
{
sourceMimetype = MIMETYPE_HTML;
targetMimetype = MIMETYPE_TEXT_PLAIN;
sourceExtension = "html";
targetExtension = "txt";
expectedOptions = null;
expectedSourceSuffix = null;
sourceFileBytes = readTestFile(sourceExtension);
expectedTargetFileBytes = Files.readAllBytes(getTestFile("quick3." + targetExtension, true).toPath());
sourceFile = new MockMultipartFile("file", "quick." + sourceExtension, sourceMimetype, sourceFileBytes);
}
@Override
// Add extra required parameters to the request.
protected MockHttpServletRequestBuilder mockMvcRequest(String url, MockMultipartFile sourceFile, String... params)
{
return super.mockMvcRequest(url, sourceFile, params)
.param("targetMimetype", targetMimetype)
.param("sourceMimetype", sourceMimetype)
.param(HTML_COLLAPSE, "true");
}
@Test
public void coreVersionNotSetInOriginalConfigTest()
{
ResponseEntity<TransformConfig> responseEntity = controller.transformConfig(Integer.valueOf(CONFIG_VERSION_DEFAULT));
responseEntity.getBody().getTransformers().forEach(transformer -> {
assertNull(transformer.getCoreVersion(), transformer.getTransformerName() +
" should have had a null coreValue but was " + transformer.getCoreVersion());
});
}
@Test
public void coreVersionSetInLatestConfigTest()
{
ResponseEntity<TransformConfig> responseEntity = controller.transformConfig(CONFIG_VERSION_LATEST);
responseEntity.getBody().getTransformers().forEach(transformer -> {
assertNotNull(transformer.getCoreVersion(), transformer.getTransformerName() +
" should have had a coreValue but was null. Should have been " + coreVersion);
});
}
@Test
public void testStartupLogsIncludeEngineMessages()
{
StringJoiner controllerLogMessages = getLogMessagesFor(TransformController.class);
controller.startup();
assertEquals(
"--------------------------------------------------------------------------------------------------------------------------------------------------------------\n"
+ "If the Alfresco software was purchased under a paid Alfresco license, the terms of the paid license agreement \n"
+ "will prevail. Otherwise, the software is provided under terms of the GNU LGPL v3 license. \n"
+ "See the license at http://www.gnu.org/licenses/lgpl-3.0.txt. or in /LICENSE.txt \n"
+ "\n"
+ "This transformer uses ImageMagick from ImageMagick Studio LLC. See the license at http://www.imagemagick.org/script/license.php or in /ImageMagick-license.txt\n"
+ "This transformer uses LibreOffice from The Document Foundation. See the license at https://www.libreoffice.org/download/license/ or in /libreoffice.txt\n"
+ "This transformer uses libraries from Apache. See the license at http://www.apache.org/licenses/LICENSE-2.0. or in /Apache\\\\ 2.0.txt\n"
+ "This transformer uses htmlparser. See the license at http://htmlparser.sourceforge.net/license.html\n"
+ "This transformer uses alfresco-pdf-renderer which uses the PDFium library from Google Inc. See the license at https://pdfium.googlesource.com/pdfium/+/master/LICENSE or in /pdfium.txt\n"
+ "This transformer uses Tika from Apache. See the license at http://www.apache.org/licenses/LICENSE-2.0. or in /Apache\\ 2.0.txt\n"
+ "This transformer uses ExifTool by Phil Harvey. See license at https://exiftool.org/#license. or in /Perl-Artistic-License.txt\n"
+ "--------------------------------------------------------------------------------------------------------------------------------------------------------------\n"
+ "Starting application components... Done",
controllerLogMessages.toString());
}
}

162
engines/aio/src/test/java/org/alfresco/transform/aio/AIOTikaTest.java
View File

@@ -1,80 +1,82 @@
/*
* #%L
* Alfresco Transform Core
* %%
* Copyright (C) 2005 - 2022 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* -
* If the software was purchased under a paid Alfresco license", "the terms of
* the paid license agreement will prevail. Otherwise", "the software is
* provided under the following open source license terms:
* -
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation", "either version 3 of the License", "or
* (at your option) any later version.
* -
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
* -
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not", "see <http://www.gnu.org/licenses/>.
* #L%
*/
package org.alfresco.transform.aio;
import com.google.common.collect.ImmutableSet;
import org.alfresco.transform.tika.TikaTest;
import org.junit.jupiter.api.Test;
import static org.alfresco.transform.base.html.OptionsHelper.getOptionNames;
import static org.junit.jupiter.api.Assertions.assertEquals;
/**
* Test Tika functionality in All-In-One.
*/
public class AIOTikaTest extends TikaTest
{
@Test
public void optionListTest()
{
assertEquals(ImmutableSet.of(
"allowEnlargement",
"allowPdfEnlargement",
"alphaRemove",
"autoOrient",
"commandOptions",
"cropGravity",
"cropHeight",
"cropPercentage",
"cropWidth",
"cropXOffset",
"cropYOffset",
"endPage",
"extractMapping",
"height",
"includeContents",
"maintainAspectRatio",
"maintainPdfAspectRatio",
"metadata",
"notExtractBookmarksText",
"page",
"pageLimit",
"pdfFormat",
"pdfOrientation",
"resizeHeight",
"resizePercentage",
"resizeWidth",
"startPage",
"targetEncoding",
"thumbnail",
"width",
"pdfFont",
"pdfFontSize"
),
getOptionNames(controller.transformConfig(0).getBody().getTransformOptions()));
}
}
/*
* #%L
* Alfresco Transform Core
* %%
* Copyright (C) 2005 - 2025 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* -
* If the software was purchased under a paid Alfresco license", "the terms of
* the paid license agreement will prevail. Otherwise", "the software is
* provided under the following open source license terms:
* -
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation", "either version 3 of the License", "or
* (at your option) any later version.
* -
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
* -
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not", "see <http://www.gnu.org/licenses/>.
* #L%
*/
package org.alfresco.transform.aio;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.alfresco.transform.base.html.OptionsHelper.getOptionNames;
import com.google.common.collect.ImmutableSet;
import org.junit.jupiter.api.Test;
import org.alfresco.transform.tika.TikaTest;
/**
* Test Tika functionality in All-In-One.
*/
public class AIOTikaTest extends TikaTest
{
@Test
public void optionListTest()
{
assertEquals(ImmutableSet.of(
"allowEnlargement",
"allowPdfEnlargement",
"alphaRemove",
"autoOrient",
"commandOptions",
"cropGravity",
"cropHeight",
"cropPercentage",
"cropWidth",
"cropXOffset",
"cropYOffset",
"endPage",
"extractMapping",
"height",
"includeContents",
"maintainAspectRatio",
"maintainPdfAspectRatio",
"metadata",
"notExtractBookmarksText",
"page",
"pageLimit",
"pdfFormat",
"pdfOrientation",
"resizeHeight",
"resizePercentage",
"resizeWidth",
"startPage",
"targetEncoding",
"thumbnail",
"width",
"pdfFont",
"pdfFontSize",
"collapseHtml"),
getOptionNames(controller.transformConfig(0).getBody().getTransformOptions()));
}
}

25
engines/misc/src/main/java/org/alfresco/transform/misc/MiscTransformEngine.java
View File

@@ -2,7 +2,7 @@
* #%L
* Alfresco Transform Core
* %%
* Copyright (C) 2005 - 2022 Alfresco Software Limited
* Copyright (C) 2005 - 2025 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* -
@@ -26,21 +26,22 @@
*/
package org.alfresco.transform.misc;
import com.google.common.collect.ImmutableMap;
import org.alfresco.transform.base.TransformEngine;
import org.alfresco.transform.base.probes.ProbeTransform;
import org.alfresco.transform.config.reader.TransformConfigResourceReader;
import org.alfresco.transform.config.TransformConfig;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.Map;
import static org.alfresco.transform.base.logging.StandardMessages.COMMUNITY_LICENCE;
import static org.alfresco.transform.common.Mimetype.MIMETYPE_HTML;
import static org.alfresco.transform.common.Mimetype.MIMETYPE_TEXT_PLAIN;
import static org.alfresco.transform.common.RequestParamMap.SOURCE_ENCODING;
import java.util.Map;
import com.google.common.collect.ImmutableMap;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.alfresco.transform.base.TransformEngine;
import org.alfresco.transform.base.probes.ProbeTransform;
import org.alfresco.transform.config.TransformConfig;
import org.alfresco.transform.config.reader.TransformConfigResourceReader;
@Component
public class MiscTransformEngine implements TransformEngine
{
@@ -74,6 +75,6 @@ public class MiscTransformEngine implements TransformEngine
public ProbeTransform getProbeTransform()
{
return new ProbeTransform("probe.html", MIMETYPE_HTML, MIMETYPE_TEXT_PLAIN, transformOptions,
119, 30, 150, 1024, 60 * 2 + 1, 60 * 2);
107, 30, 150, 1024, 60 * 2 + 1, 60 * 2);
}
}

418
engines/misc/src/main/java/org/alfresco/transform/misc/transformers/HtmlParserContentTransformer.java
View File

@@ -1,203 +1,215 @@
/*
* #%L
* Alfresco Transform Core
* %%
* Copyright (C) 2005 - 2022 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* -
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
* -
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
* -
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
* -
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
package org.alfresco.transform.misc.transformers;
import org.alfresco.transform.base.TransformManager;
import org.alfresco.transform.base.util.CustomTransformerFileAdaptor;
import org.htmlparser.Parser;
import org.htmlparser.beans.StringBean;
import org.htmlparser.util.ParserException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.net.URLConnection;
import java.nio.charset.Charset;
import java.nio.charset.IllegalCharsetNameException;
import java.util.Map;
import static org.alfresco.transform.common.RequestParamMap.SOURCE_ENCODING;
/**
* Content transformer which wraps the HTML Parser library for
* parsing HTML content.
*
* <p>
* This code is based on a class of the same name originally implemented in alfresco-repository.
* </p>
*
* <p>
* Since HTML Parser was updated from v1.6 to v2.1, META tags
* defining an encoding for the content via http-equiv=Content-Type
* will ONLY be respected if the encoding of the content item
* itself is set to ISO-8859-1.
* </p>
*
* <p>
* Tika Note - could be converted to use the Tika HTML parser,
* but we'd potentially need a custom text handler to replicate
* the current settings around links and non-breaking spaces.
* </p>
*
* @author Derek Hulley
* @author eknizat
* @see <a href="http://htmlparser.sourceforge.net/">http://htmlparser.sourceforge.net</a>
* @see org.htmlparser.beans.StringBean
* @see <a href="http://sourceforge.net/tracker/?func=detail&aid=1644504&group_id=24399&atid=381401">HTML Parser</a>
*/
@Component
public class HtmlParserContentTransformer implements CustomTransformerFileAdaptor
{
private static final Logger logger = LoggerFactory.getLogger(
HtmlParserContentTransformer.class);
@Override
public String getTransformerName()
{
return "html";
}
@Override
public void transform(final String sourceMimetype, final String targetMimetype,
final Map<String, String> transformOptions,
final File sourceFile, final File targetFile, TransformManager transformManager) throws Exception
{
String sourceEncoding = transformOptions.get(SOURCE_ENCODING);
checkEncodingParameter(sourceEncoding, SOURCE_ENCODING);
if (logger.isDebugEnabled())
{
logger.debug("Performing HTML to text transform with sourceEncoding=" + sourceEncoding);
}
// Create the extractor
EncodingAwareStringBean extractor = new EncodingAwareStringBean();
extractor.setCollapse(false);
extractor.setLinks(false);
extractor.setReplaceNonBreakingSpaces(false);
extractor.setURL(sourceFile, sourceEncoding);
// get the text
String text = extractor.getStrings();
// write it to the writer
try (Writer writer = new BufferedWriter(
new OutputStreamWriter(new FileOutputStream(targetFile))))
{
writer.write(text);
}
}
private void checkEncodingParameter(String encoding, String parameterName)
{
try
{
if (encoding != null && !Charset.isSupported(encoding))
{
throw new IllegalArgumentException(
parameterName + "=" + encoding + " is not supported by the JVM.");
}
}
catch (IllegalCharsetNameException e)
{
throw new IllegalArgumentException(
parameterName + "=" + encoding + " is not a valid encoding.");
}
}
/**
* <p>
* This code is based on a class of the same name, originally implemented in alfresco-repository.
* </p>
*
* A version of {@link StringBean} which allows control of the
* encoding in the underlying HTML Parser.
* Unfortunately, StringBean doesn't allow easy over-riding of
* this, so we have to duplicate some code to control this.
* This allows us to correctly handle HTML files where the encoding
* is specified against the content property (rather than in the
* HTML Head Meta), see ALF-10466 for details.
*/
public static class EncodingAwareStringBean extends StringBean
{
private static final long serialVersionUID = -9033414360428669553L;
/**
* Sets the File to extract strings from, and the encoding
* it's in (if known to Alfresco)
*
* @param file The File that text should be fetched from.
* @param encoding The encoding of the input
*/
public void setURL(File file, String encoding)
{
String previousURL = getURL();
String newURL = file.getAbsolutePath();
if (previousURL == null || !newURL.equals(previousURL))
{
try
{
URLConnection conn = getConnection();
if (null == mParser)
{
mParser = new Parser(newURL);
}
else
{
mParser.setURL(newURL);
}
if (encoding != null)
{
mParser.setEncoding(encoding);
}
mPropertySupport.firePropertyChange(StringBean.PROP_URL_PROPERTY, previousURL,
getURL());
mPropertySupport.firePropertyChange(StringBean.PROP_CONNECTION_PROPERTY, conn,
mParser.getConnection());
setStrings();
}
catch (ParserException pe)
{
updateStrings(pe.toString());
}
}
}
public String getEncoding()
{
return mParser.getEncoding();
}
}
}
/*
* #%L
* Alfresco Transform Core
* %%
* Copyright (C) 2005 - 2025 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* -
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
* -
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
* -
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
* -
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
package org.alfresco.transform.misc.transformers;
import static org.alfresco.transform.common.RequestParamMap.HTML_COLLAPSE;
import static org.alfresco.transform.common.RequestParamMap.SOURCE_ENCODING;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.net.URLConnection;
import java.nio.charset.Charset;
import java.nio.charset.IllegalCharsetNameException;
import java.util.Map;
import org.htmlparser.Parser;
import org.htmlparser.beans.StringBean;
import org.htmlparser.util.ParserException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.alfresco.transform.base.TransformManager;
import org.alfresco.transform.base.util.CustomTransformerFileAdaptor;
/**
* Content transformer which wraps the HTML Parser library for parsing HTML content.
*
* <p>
* This code is based on a class of the same name originally implemented in alfresco-repository.
* </p>
*
* <p>
* Since HTML Parser was updated from v1.6 to v2.1, META tags defining an encoding for the content via http-equiv=Content-Type will ONLY be respected if the encoding of the content item itself is set to ISO-8859-1.
* </p>
*
* <p>
* Tika Note - could be converted to use the Tika HTML parser, but we'd potentially need a custom text handler to replicate the current settings around links and non-breaking spaces.
* </p>
*
* @author Derek Hulley
* @author eknizat
* @see <a href="http://htmlparser.sourceforge.net/">http://htmlparser.sourceforge.net</a>
* @see org.htmlparser.beans.StringBean
* @see <a href="http://sourceforge.net/tracker/?func=detail&aid=1644504&group_id=24399&atid=381401">HTML Parser</a>
*/
@Component
public class HtmlParserContentTransformer implements CustomTransformerFileAdaptor
{
private static final Logger logger = LoggerFactory.getLogger(
HtmlParserContentTransformer.class);
@Value("${transform.core.misc.htmlOptions.collapseHtml:true}")
private String collapseOptionDefault;
@Override
public String getTransformerName()
{
return "html";
}
@Override
public void transform(final String sourceMimetype, final String targetMimetype,
final Map<String, String> transformOptions,
final File sourceFile, final File targetFile, TransformManager transformManager) throws Exception
{
String sourceEncoding = transformOptions.get(SOURCE_ENCODING);
checkEncodingParameter(sourceEncoding, SOURCE_ENCODING);
boolean collapse;
var collapseOption = transformOptions.get(HTML_COLLAPSE);
// If the collapse option is set, use it, otherwise use the default value
if (collapseOption != null && (collapseOption.trim().equalsIgnoreCase("true") || collapseOption.trim().equalsIgnoreCase("false")))
{
collapse = Boolean.parseBoolean(collapseOption);
}
else
{
// Use the default value from the configuration
collapse = collapseOptionDefault == null || Boolean.parseBoolean(collapseOptionDefault);
if (logger.isDebugEnabled())
{
logger.debug("Using default html collapse option: " + collapseOptionDefault);
}
}
if (logger.isDebugEnabled())
{
logger.debug("Performing HTML to text transform with sourceEncoding=" + sourceEncoding);
}
// Create the extractor
EncodingAwareStringBean extractor = new EncodingAwareStringBean();
extractor.setCollapse(collapse);
extractor.setLinks(false);
extractor.setReplaceNonBreakingSpaces(false);
extractor.setURL(sourceFile, sourceEncoding);
// get the text
String text = extractor.getStrings();
// write it to the writer
try (Writer writer = new BufferedWriter(
new OutputStreamWriter(new FileOutputStream(targetFile))))
{
writer.write(text);
}
}
private void checkEncodingParameter(String encoding, String parameterName)
{
try
{
if (encoding != null && !Charset.isSupported(encoding))
{
throw new IllegalArgumentException(
parameterName + "=" + encoding + " is not supported by the JVM.");
}
}
catch (IllegalCharsetNameException e)
{
throw new IllegalArgumentException(
parameterName + "=" + encoding + " is not a valid encoding.");
}
}
/**
* <p>
* This code is based on a class of the same name, originally implemented in alfresco-repository.
* </p>
*
* A version of {@link StringBean} which allows control of the encoding in the underlying HTML Parser. Unfortunately, StringBean doesn't allow easy over-riding of this, so we have to duplicate some code to control this. This allows us to correctly handle HTML files where the encoding is specified against the content property (rather than in the HTML Head Meta), see ALF-10466 for details.
*/
public static class EncodingAwareStringBean extends StringBean
{
private static final long serialVersionUID = -9033414360428669553L;
/**
* Sets the File to extract strings from, and the encoding it's in (if known to Alfresco)
*
* @param file
* The File that text should be fetched from.
* @param encoding
* The encoding of the input
*/
public void setURL(File file, String encoding)
{
String previousURL = getURL();
String newURL = file.getAbsolutePath();
if (previousURL == null || !newURL.equals(previousURL))
{
try
{
URLConnection conn = getConnection();
if (null == mParser)
{
mParser = new Parser(newURL);
}
else
{
mParser.setURL(newURL);
}
if (encoding != null)
{
mParser.setEncoding(encoding);
}
mPropertySupport.firePropertyChange(StringBean.PROP_URL_PROPERTY, previousURL,
getURL());
mPropertySupport.firePropertyChange(StringBean.PROP_CONNECTION_PROPERTY, conn,
mParser.getConnection());
setStrings();
}
catch (ParserException pe)
{
updateStrings(pe.toString());
}
}
}
public String getEncoding()
{
return mParser.getEncoding();
}
}
}

4
engines/misc/src/main/resources/application-default.yaml
View File

@@ -4,4 +4,6 @@ transform:
core:
misc:
pdfBox:
defaultFont: ${MISC_PDFBOX_DEFAULT_FONT:NotoSans-Regular}
defaultFont: ${MISC_PDFBOX_DEFAULT_FONT:NotoSans-Regular}
htmlOptions:
collapseHtml: ${MISC_HTML_COLLAPSE:true}

6
engines/misc/src/main/resources/misc_engine_config.json
View File

@@ -1,5 +1,8 @@
{
"transformOptions": {
"htmlOptions": [
{"value": {"name": "collapseHtml"}}
],
"textToPdfOptions": [
{"value": {"name": "pageLimit"}},
{"value": {"name": "pdfFont"}},
@@ -24,8 +27,7 @@
"supportedSourceAndTargetList": [
{"sourceMediaType": "text/html", "targetMediaType": "text/plain"}
],
"transformOptions": [
]
"transformOptions": ["htmlOptions"]
},
{
"transformerName": "string",

1002
engines/misc/src/test/java/org/alfresco/transform/misc/MiscTest.java
View File

File diff suppressed because it is too large Load Diff

462
engines/misc/src/test/java/org/alfresco/transform/misc/transformers/HtmlParserContentTransformerTest.java
View File

@@ -1,162 +1,300 @@
/*
* #%L
* Alfresco Transform Core
* %%
* Copyright (C) 2005 - 2022 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* -
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
* -
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
* -
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
* -
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
package org.alfresco.transform.misc.transformers;
import org.junit.jupiter.api.Test;
import java.io.File;
import java.io.FileOutputStream;
import java.io.OutputStreamWriter;
import java.nio.file.Files;
import java.util.HashMap;
import java.util.Map;
import static org.alfresco.transform.common.RequestParamMap.SOURCE_ENCODING;
import static org.junit.jupiter.api.Assertions.assertEquals;
public class HtmlParserContentTransformerTest
{
private static final String SOURCE_MIMETYPE = "text/html";
private static final String TARGET_MIMETYPE = "text/plain";
HtmlParserContentTransformer transformer = new HtmlParserContentTransformer();
/**
* Checks that we correctly handle text in different encodings,
* no matter if the encoding is specified on the Content Property
* or in a meta tag within the HTML itself. (ALF-10466)
*
* On Windows, org.htmlparser.beans.StringBean.carriageReturn() appends a new system dependent new line
* so we must be careful when checking the returned text
*/
@Test
public void testEncodingHandling() throws Exception
{
final String NEWLINE = System.getProperty("line.separator");
final String TITLE = "Testing!";
final String TEXT_P1 = "This is some text in English";
final String TEXT_P2 = "This is more text in English";
final String TEXT_P3 = "C'est en Fran\u00e7ais et Espa\u00f1ol";
String partA = "<html><head><title>" + TITLE + "</title></head>" + NEWLINE;
String partB = "<body><p>" + TEXT_P1 + "</p>" + NEWLINE +
"<p>" + TEXT_P2 + "</p>" + NEWLINE +
"<p>" + TEXT_P3 + "</p>" + NEWLINE;
String partC = "</body></html>";
final String expected = TITLE + NEWLINE + TEXT_P1 + NEWLINE + TEXT_P2 + NEWLINE + TEXT_P3 + NEWLINE;
File tmpS = null;
File tmpD = null;
try
{
// Content set to ISO 8859-1
tmpS = File.createTempFile("AlfrescoTestSource_", ".html");
writeToFile(tmpS, partA + partB + partC, "ISO-8859-1");
tmpD = File.createTempFile("AlfrescoTestTarget_", ".txt");
Map<String, String> parameters = new HashMap<>();
parameters.put(SOURCE_ENCODING, "ISO-8859-1");
transformer.transform(SOURCE_MIMETYPE, TARGET_MIMETYPE, parameters, tmpS, tmpD, null);
assertEquals(expected, readFromFile(tmpD, "UTF-8"));
tmpS.delete();
tmpD.delete();
// Content set to UTF-8
tmpS = File.createTempFile("AlfrescoTestSource_", ".html");
writeToFile(tmpS, partA + partB + partC, "UTF-8");
tmpD = File.createTempFile("AlfrescoTestTarget_", ".txt");
parameters = new HashMap<>();
parameters.put(SOURCE_ENCODING, "UTF-8");
transformer.transform(SOURCE_MIMETYPE, TARGET_MIMETYPE, parameters, tmpS, tmpD, null);
assertEquals(expected, readFromFile(tmpD, "UTF-8"));
tmpS.delete();
tmpD.delete();
// Content set to UTF-16
tmpS = File.createTempFile("AlfrescoTestSource_", ".html");
writeToFile(tmpS, partA + partB + partC, "UTF-16");
tmpD = File.createTempFile("AlfrescoTestTarget_", ".txt");
parameters = new HashMap<>();
parameters.put(SOURCE_ENCODING, "UTF-16");
transformer.transform(SOURCE_MIMETYPE, TARGET_MIMETYPE, parameters, tmpS, tmpD, null);
assertEquals(expected, readFromFile(tmpD, "UTF-8"));
tmpS.delete();
tmpD.delete();
// Note - since HTML Parser 2.0 META tags specifying the
// document encoding will ONLY be respected if the original
// content type was set to ISO-8859-1.
//
// This means there is now only one test which we can perform
// to ensure that this now-limited overriding of the encoding
// takes effect.
// Content set to ISO 8859-1, meta set to UTF-8
tmpS = File.createTempFile("AlfrescoTestSource_", ".html");
String str = partA +
"<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\">" +
partB + partC;
writeToFile(tmpS, str, "UTF-8");
tmpD = File.createTempFile("AlfrescoTestTarget_", ".txt");
parameters = new HashMap<>();
parameters.put(SOURCE_ENCODING, "ISO-8859-1");
transformer.transform(SOURCE_MIMETYPE, TARGET_MIMETYPE, parameters, tmpS, tmpD, null);
assertEquals(expected, readFromFile(tmpD, "UTF-8"));
tmpS.delete();
tmpD.delete();
// Note - we can't test UTF-16 with only a meta encoding,
// because without that the parser won't know about the
// 2 byte format so won't be able to identify the meta tag
}
finally
{
if (tmpS != null && tmpS.exists()) tmpS.delete();
if (tmpD != null && tmpD.exists()) tmpD.delete();
}
}
private void writeToFile(File file, String content, String encoding) throws Exception
{
try (OutputStreamWriter ow = new OutputStreamWriter(new FileOutputStream(file), encoding))
{
ow.append(content);
}
}
private String readFromFile(File file, final String encoding) throws Exception
{
return new String(Files.readAllBytes(file.toPath()), encoding);
}
}
/*
* #%L
* Alfresco Transform Core
* %%
* Copyright (C) 2005 - 2025 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* -
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
* -
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
* -
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
* -
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
package org.alfresco.transform.misc.transformers;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.fail;
import static org.alfresco.transform.common.RequestParamMap.HTML_COLLAPSE;
import static org.alfresco.transform.common.RequestParamMap.SOURCE_ENCODING;
import java.io.File;
import java.io.FileOutputStream;
import java.io.OutputStreamWriter;
import java.nio.file.Files;
import java.util.HashMap;
import java.util.Map;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
public class HtmlParserContentTransformerTest
{
private static final String SOURCE_MIMETYPE = "text/html";
private static final String TARGET_MIMETYPE = "text/plain";
/**
* Checks that we correctly handle text in different encodings, no matter if the encoding is specified on the Content Property or in a meta tag within the HTML itself. (ALF-10466)
*
* On Windows, org.htmlparser.beans.StringBean.carriageReturn() appends a new system dependent new line so we must be careful when checking the returned text
*/
@Test
public void testEncodingHandling() throws Exception
{
final HtmlParserContentTransformer transformer = new HtmlParserContentTransformer();
final String newline = System.getProperty("line.separator");
final String title = "Testing!";
final String textp1 = "This is some text in English";
final String textp2 = "This is more text in English";
final String textp3 = "C'est en Fran\u00e7ais et Espa\u00f1ol";
String partA = "<html><head><title>" + title + "</title></head>" + newline;
String partB = "<body><p>" + textp1 + "</p>" + newline +
"<p>" + textp2 + "</p>" + newline +
"<p>" + textp3 + "</p>" + newline;
String partC = "</body></html>";
final String expected = title + newline + textp1 + newline + textp2 + newline + textp3;
File tmpS = null;
File tmpD = null;
try
{
// Content set to ISO 8859-1
tmpS = File.createTempFile("AlfrescoTestSource_", ".html");
writeToFile(tmpS, partA + partB + partC, "ISO-8859-1");
tmpD = File.createTempFile("AlfrescoTestTarget_", ".txt");
Map<String, String> parameters = new HashMap<>();
parameters.put(SOURCE_ENCODING, "ISO-8859-1");
parameters.put(HTML_COLLAPSE, String.valueOf(true));
transformer.transform(SOURCE_MIMETYPE, TARGET_MIMETYPE, parameters, tmpS, tmpD, null);
assertEquals(expected, readFromFile(tmpD, "UTF-8"));
tmpS.delete();
tmpD.delete();
// Content set to UTF-8
tmpS = File.createTempFile("AlfrescoTestSource_", ".html");
writeToFile(tmpS, partA + partB + partC, "UTF-8");
tmpD = File.createTempFile("AlfrescoTestTarget_", ".txt");
parameters = new HashMap<>();
parameters.put(SOURCE_ENCODING, "UTF-8");
parameters.put(HTML_COLLAPSE, String.valueOf(true));
transformer.transform(SOURCE_MIMETYPE, TARGET_MIMETYPE, parameters, tmpS, tmpD, null);
assertEquals(expected, readFromFile(tmpD, "UTF-8"));
tmpS.delete();
tmpD.delete();
// Content set to UTF-16
tmpS = File.createTempFile("AlfrescoTestSource_", ".html");
writeToFile(tmpS, partA + partB + partC, "UTF-16");
tmpD = File.createTempFile("AlfrescoTestTarget_", ".txt");
parameters = new HashMap<>();
parameters.put(HTML_COLLAPSE, String.valueOf(true));
parameters.put(SOURCE_ENCODING, "UTF-16");
transformer.transform(SOURCE_MIMETYPE, TARGET_MIMETYPE, parameters, tmpS, tmpD, null);
assertEquals(expected, readFromFile(tmpD, "UTF-8"));
tmpS.delete();
tmpD.delete();
// Note - since HTML Parser 2.0 META tags specifying the
// document encoding will ONLY be respected if the original
// content type was set to ISO-8859-1.
//
// This means there is now only one test which we can perform
// to ensure that this now-limited overriding of the encoding
// takes effect.
// Content set to ISO 8859-1, meta set to UTF-8
tmpS = File.createTempFile("AlfrescoTestSource_", ".html");
String str = partA +
"<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\">" +
partB + partC;
writeToFile(tmpS, str, "UTF-8");
tmpD = File.createTempFile("AlfrescoTestTarget_", ".txt");
parameters = new HashMap<>();
parameters.put(SOURCE_ENCODING, "ISO-8859-1");
parameters.put(HTML_COLLAPSE, String.valueOf(true));
transformer.transform(SOURCE_MIMETYPE, TARGET_MIMETYPE, parameters, tmpS, tmpD, null);
assertEquals(expected, readFromFile(tmpD, "UTF-8"));
tmpS.delete();
tmpD.delete();
// Note - we can't test UTF-16 with only a meta encoding,
// because without that the parser won't know about the
// 2 byte format so won't be able to identify the meta tag
}
catch (Exception e)
{
fail("Test Failed: " + e.getMessage()); // fail the test if any exception occurs
}
finally
{
if (tmpS != null && tmpS.exists())
{
tmpS.delete();
}
if (tmpD != null && tmpD.exists())
{
tmpD.delete();
}
}
}
/**
* Tests the transformer with different collapsing methods. If the collapsing is set to false, it should not collapse the new lines between paragraphs. If the collapsing is set to true, it should collapse the new lines.
*/
@ParameterizedTest
@ValueSource(booleans = {true, false})
public void testTransformerWithDifferentCollapsingMethods(boolean shouldCollapse)
{
final HtmlParserContentTransformer transformer = new HtmlParserContentTransformer();
final String newline = System.getProperty("line.separator");
final String title = "Testing!";
final String textp1 = "This is some text in English";
final String textp2 = "This is more text in English";
final String textp3 = "C'est en Fran\u00e7ais et Espa\u00f1ol";
String partA = "<html><head><title>" + title + "</title></head>" + newline;
String partB = "<body><p>" + textp1 + "</p>" + newline +
"<p>" + textp2 + "</p>" + newline +
"<p>" + textp3 + "</p>" + newline;
String partC = "</body></html>";
final String expected = title + newline + textp1 + newline + textp2 + newline + textp3 + (shouldCollapse ? "" : newline); // Just a added newline if collapsing is not collapsing
File tmpS = null;
File tmpD = null;
try
{
tmpS = File.createTempFile("AlfrescoTestSource_", ".html");
writeToFile(tmpS, partA + partB + partC, "UTF-8");
tmpD = File.createTempFile("AlfrescoTestTarget_", ".txt");
Map<String, String> parameters = new HashMap<>();
parameters.put(SOURCE_ENCODING, "UTF-8");
parameters.put(HTML_COLLAPSE, String.valueOf(shouldCollapse));
transformer.transform(SOURCE_MIMETYPE, TARGET_MIMETYPE, parameters, tmpS, tmpD, null);
assertEquals(expected, readFromFile(tmpD, "UTF-8"));
tmpS.delete();
tmpD.delete();
}
catch (Exception e)
{
fail("Test Failed: " + e.getMessage()); // fail the test if any exception occurs
}
finally
{
if (tmpS != null && tmpS.exists())
{
tmpS.delete();
}
if (tmpD != null && tmpD.exists())
{
tmpD.delete();
}
}
}
/**
* Tests the transformer with wrong boolean values for the collapse option. It should not throw an exception and should use the default value for collapsing.
*/
@ParameterizedTest
@ValueSource(strings = {"cat", "dog", "", "1234abcd", "@#$%"})
public void testTransformerWithWrongBooleanValues(String booleanValues)
{
final HtmlParserContentTransformer transformer = new HtmlParserContentTransformer();
final String newline = System.getProperty("line.separator");
final String title = "Testing!";
final String textp1 = "This is some text in English";
final String textp2 = "This is more text in English";
final String textp3 = "C'est en Fran\u00e7ais et Espa\u00f1ol";
String partA = "<html><head><title>" + title + "</title></head>" + newline;
String partB = "<body><p>" + textp1 + "</p>" + newline +
"<p>" + textp2 + "</p>" + newline +
"<p>" + textp3 + "</p>" + newline;
String partC = "</body></html>";
final String expected = title + newline + textp1 + newline + textp2 + newline + textp3;
File tmpS = null;
File tmpD = null;
try
{
tmpS = File.createTempFile("AlfrescoTestSource_", ".html");
writeToFile(tmpS, partA + partB + partC, "UTF-8");
tmpD = File.createTempFile("AlfrescoTestTarget_", ".txt");
Map<String, String> parameters = new HashMap<>();
parameters.put(SOURCE_ENCODING, "UTF-8");
parameters.put(HTML_COLLAPSE, booleanValues);
transformer.transform(SOURCE_MIMETYPE, TARGET_MIMETYPE, parameters, tmpS, tmpD, null);
assertEquals(expected, readFromFile(tmpD, "UTF-8"));
tmpS.delete();
tmpD.delete();
}
catch (Exception e)
{
fail("Test Failed: " + e.getMessage()); // fail the test if any exception occurs
}
finally
{
if (tmpS != null && tmpS.exists())
{
tmpS.delete();
}
if (tmpD != null && tmpD.exists())
{
tmpD.delete();
}
}
}
private void writeToFile(File file, String content, String encoding)
{
try (OutputStreamWriter ow = new OutputStreamWriter(new FileOutputStream(file), encoding))
{
ow.append(content);
}
catch (Exception e)
{
fail("Failed to write to file: " + e.getMessage()); // fail the test if any exception occurs
}
}
private String readFromFile(File file, final String encoding)
{
try
{
return new String(Files.readAllBytes(file.toPath()), encoding);
}
catch (Exception e)
{
fail("Failed to read from file: " + e.getMessage());
return null; // Return null if there is an error reading the file
}
}
}

4
engines/misc/src/test/resources/misc_engine_config.json
View File

@@ -1,5 +1,8 @@
{
"transformOptions": {
"htmlOptions": [
{"value": {"name": "collapseHtml"}}
],
"textToPdfOptions": [
{"value": {"name": "pageLimit"}}
],
@@ -17,6 +20,7 @@
{"sourceMediaType": "text/html", "targetMediaType": "text/plain"}
],
"transformOptions": [
"htmlOptions"
]
},
{

2
engines/misc/src/test/resources/quick3.txt Normal file
View File

@@ -0,0 +1,2 @@
The quick brown fox jumps over the lazy dog
The quick brown fox jumps over the lazy dog

284
engines/tika/src/test/java/org/alfresco/transform/tika/TikaTest.java
View File

@@ -2,7 +2,7 @@
* #%L
* Alfresco Transform Core
* %%
* Copyright (C) 2005 - 2022 Alfresco Software Limited
* Copyright (C) 2005 - 2025 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* -
@@ -26,31 +26,21 @@
*/
package org.alfresco.transform.tika;
import com.google.common.collect.ImmutableSet;
import org.alfresco.transform.base.AbstractBaseTest;
import org.alfresco.transform.base.executors.RuntimeExec;
import org.alfresco.transform.base.model.FileRefEntity;
import org.alfresco.transform.base.model.FileRefResponse;
import org.alfresco.transform.client.model.TransformReply;
import org.alfresco.transform.client.model.TransformRequest;
import org.apache.poi.ooxml.POIXMLProperties;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mock;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.Resource;
import org.springframework.http.HttpHeaders;
import org.springframework.http.ResponseEntity;
import org.springframework.mock.web.MockMultipartFile;
import org.springframework.test.web.servlet.MvcResult;
import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder;
import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.util.UUID;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
import static org.springframework.http.HttpHeaders.ACCEPT;
import static org.springframework.http.HttpHeaders.CONTENT_DISPOSITION;
import static org.springframework.http.HttpHeaders.CONTENT_TYPE;
import static org.springframework.http.HttpStatus.CREATED;
import static org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR;
import static org.springframework.http.HttpStatus.OK;
import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE;
import static org.springframework.http.MediaType.APPLICATION_PDF_VALUE;
import static org.springframework.http.MediaType.TEXT_PLAIN_VALUE;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import static org.alfresco.transform.base.html.OptionsHelper.getOptionNames;
import static org.alfresco.transform.common.Mimetype.MIMETYPE_HTML;
@@ -89,21 +79,33 @@ import static org.alfresco.transform.tika.transformers.Tika.XHTML;
import static org.alfresco.transform.tika.transformers.Tika.XLSX;
import static org.alfresco.transform.tika.transformers.Tika.XML;
import static org.alfresco.transform.tika.transformers.Tika.ZIP;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
import static org.springframework.http.HttpHeaders.ACCEPT;
import static org.springframework.http.HttpHeaders.CONTENT_DISPOSITION;
import static org.springframework.http.HttpHeaders.CONTENT_TYPE;
import static org.springframework.http.HttpStatus.CREATED;
import static org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR;
import static org.springframework.http.HttpStatus.OK;
import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE;
import static org.springframework.http.MediaType.APPLICATION_PDF_VALUE;
import static org.springframework.http.MediaType.TEXT_PLAIN_VALUE;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.util.UUID;
import com.google.common.collect.ImmutableSet;
import org.apache.poi.ooxml.POIXMLProperties;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mock;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.Resource;
import org.springframework.http.HttpHeaders;
import org.springframework.http.ResponseEntity;
import org.springframework.mock.web.MockMultipartFile;
import org.springframework.test.web.servlet.MvcResult;
import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder;
import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
import org.alfresco.transform.base.AbstractBaseTest;
import org.alfresco.transform.base.executors.RuntimeExec;
import org.alfresco.transform.base.model.FileRefEntity;
import org.alfresco.transform.base.model.FileRefResponse;
import org.alfresco.transform.client.model.TransformReply;
import org.alfresco.transform.client.model.TransformRequest;
/**
* Test Tika.
@@ -113,9 +115,9 @@ public class TikaTest extends AbstractBaseTest
private static final String EXPECTED_XHTML_CONTENT_CONTAINS = "<p>The quick brown fox jumps over the lazy dog</p>";
private static final String EXPECTED_TEXT_CONTENT_CONTAINS = "The quick brown fox jumps over the lazy dog";
private static final String EXPECTED_MSG_CONTENT_CONTAINS = "Recipients\n" +
"\tmark.rogers@alfresco.com; speedy@quick.com; mrquick@nowhere.com\n" +
"\n" +
"The quick brown fox jumps over the lazy dogs";
"\tmark.rogers@alfresco.com; speedy@quick.com; mrquick@nowhere.com\n" +
"\n" +
"The quick brown fox jumps over the lazy dogs";
private static final String EXPECTED_CSV_CONTENT_CONTAINS = "\"The\",\"quick\",\"brown\",\"fox\"";
@Mock
@@ -139,8 +141,8 @@ public class TikaTest extends AbstractBaseTest
@Override
protected void mockTransformCommand(String sourceExtension,
String targetExtension, String sourceMimetype,
boolean readTargetFileBytes) throws IOException
String targetExtension, String sourceMimetype,
boolean readTargetFileBytes) throws IOException
{
// Tika transform is not mocked. It is run for real.
@@ -160,8 +162,8 @@ public class TikaTest extends AbstractBaseTest
}
private void transform(String transform, String sourceExtension, String targetExtension,
String sourceMimetype, String targetMimetype,
Boolean includeContents, String expectedContentContains) throws Exception
String sourceMimetype, String targetMimetype,
Boolean includeContents, String expectedContentContains) throws Exception
{
// We don't use targetFileBytes as some of the transforms contain different date text based on the os being used.
mockTransformCommand(sourceExtension, targetExtension, sourceMimetype, false);
@@ -169,18 +171,18 @@ public class TikaTest extends AbstractBaseTest
System.out.println("Test " + transform + " " + sourceExtension + " to " + targetExtension);
MockHttpServletRequestBuilder requestBuilder = includeContents == null
? mockMvcRequest(ENDPOINT_TRANSFORM, sourceFile,
"targetExtension", this.targetExtension)
: mockMvcRequest(ENDPOINT_TRANSFORM, sourceFile,
"targetExtension", this.targetExtension, INCLUDE_CONTENTS, includeContents.toString());
? mockMvcRequest(ENDPOINT_TRANSFORM, sourceFile,
"targetExtension", this.targetExtension)
: mockMvcRequest(ENDPOINT_TRANSFORM, sourceFile,
"targetExtension", this.targetExtension, INCLUDE_CONTENTS, includeContents.toString());
MvcResult result = mockMvc.perform(requestBuilder)
.andExpect(status().is(OK.value()))
.andExpect(header().string("Content-Disposition",
"attachment; filename*=UTF-8''transform." + this.targetExtension))
.andReturn();
.andExpect(status().is(OK.value()))
.andExpect(header().string("Content-Disposition",
"attachment; filename*=UTF-8''transform." + this.targetExtension))
.andReturn();
String content = result.getResponse().getContentAsString();
assertTrue(content.contains(expectedContentContains),
"The content did not include \"" + expectedContentContains);
"The content did not include \"" + expectedContentContains);
}
@Override
@@ -188,9 +190,9 @@ public class TikaTest extends AbstractBaseTest
protected MockHttpServletRequestBuilder mockMvcRequest(String url, MockMultipartFile sourceFile, String... params)
{
return super.mockMvcRequest(url, sourceFile, params)
.param("targetEncoding", targetEncoding)
.param("targetMimetype", targetMimetype)
.param("sourceMimetype", sourceMimetype);
.param("targetEncoding", targetEncoding)
.param("targetMimetype", targetMimetype)
.param("sourceMimetype", sourceMimetype);
}
@Test
@@ -199,8 +201,8 @@ public class TikaTest extends AbstractBaseTest
mockTransformCommand(PDF, TXT, MIMETYPE_PDF, true);
targetEncoding = "rubbish";
mockMvc.perform(
mockMvcRequest(ENDPOINT_TRANSFORM, sourceFile, "targetExtension", targetExtension))
.andExpect(status().is(INTERNAL_SERVER_ERROR.value()));
mockMvcRequest(ENDPOINT_TRANSFORM, sourceFile, "targetExtension", targetExtension))
.andExpect(status().is(INTERNAL_SERVER_ERROR.value()));
}
// --- Archive ---
@@ -209,55 +211,55 @@ public class TikaTest extends AbstractBaseTest
public void zipToTextArchiveTest() throws Exception
{
transform(ARCHIVE, ZIP, TXT, MIMETYPE_ZIP, MIMETYPE_TEXT_PLAIN, false,
"quick.html\n" +
"\n" +
"\n" +
"quick.pdf\n" +
"\n" +
"\n");
"quick.html\n" +
"\n" +
"\n" +
"quick.pdf\n" +
"\n" +
"\n");
}
@Test
public void zipToTextIncludeArchiveTest() throws Exception
{
transform(ARCHIVE, ZIP, TXT, MIMETYPE_ZIP, MIMETYPE_TEXT_PLAIN, true,
"quick.html\n" +
"\n" +
"\n" +
"The quick brown fox jumps over the lazy dog\n" +
"\n" +
"\n" +
"\n" +
"quick.pdf\n" +
"\n" +
"\n" +
"The quick brown fox jumps over the lazy dog" +
"\n" +
"\n");
"quick.html\n" +
"\n" +
"\n" +
"The quick brown fox jumps over the lazy dog\n" +
"\n" +
"\n" +
"\n" +
"quick.pdf\n" +
"\n" +
"\n" +
"The quick brown fox jumps over the lazy dog" +
"\n" +
"\n");
}
@Test
public void zipToTextExcludeArchiveTest() throws Exception
{
transform(ARCHIVE, ZIP, TXT, MIMETYPE_ZIP, MIMETYPE_TEXT_PLAIN,
false, "\n" +
"folder/subfolder/quick.jpg\n" +
"\n" +
"\n" +
"quick.doc\n" +
"\n" +
"\n" +
"quick.html\n" +
"\n" +
"\n" +
"quick.pdf\n" +
"\n" +
"\n" +
"quick.txt\n" +
"\n" +
"\n" +
"quick.xml\n" +
"\n");
false, "\n" +
"folder/subfolder/quick.jpg\n" +
"\n" +
"\n" +
"quick.doc\n" +
"\n" +
"\n" +
"quick.html\n" +
"\n" +
"\n" +
"quick.pdf\n" +
"\n" +
"\n" +
"quick.txt\n" +
"\n" +
"\n" +
"quick.xml\n" +
"\n");
}
// --- OutlookMsg ---
@@ -266,7 +268,7 @@ public class TikaTest extends AbstractBaseTest
public void msgToTxtOutlookMsgTest() throws Exception
{
transform(OUTLOOK_MSG, MSG, TXT, MIMETYPE_OUTLOOK_MSG, MIMETYPE_TEXT_PLAIN, null,
EXPECTED_MSG_CONTENT_CONTAINS);
EXPECTED_MSG_CONTENT_CONTAINS);
}
// --- PdfBox ---
@@ -275,35 +277,35 @@ public class TikaTest extends AbstractBaseTest
public void pdfToTxtPdfBoxTest() throws Exception
{
transform(PDF_BOX, PDF, TXT, MIMETYPE_PDF, MIMETYPE_TEXT_PLAIN, null,
EXPECTED_TEXT_CONTENT_CONTAINS);
EXPECTED_TEXT_CONTENT_CONTAINS);
}
@Test
public void pdfToCsvPdfBoxTest() throws Exception
{
transform(PDF_BOX, PDF, CSV, MIMETYPE_PDF, MIMETYPE_TEXT_CSV, null,
EXPECTED_TEXT_CONTENT_CONTAINS); // Yes it is just text
EXPECTED_TEXT_CONTENT_CONTAINS); // Yes it is just text
}
@Test
public void pdfToXmlPdfBoxTest() throws Exception
{
transform(PDF_BOX, PDF, XML, MIMETYPE_PDF, MIMETYPE_XML, null,
EXPECTED_XHTML_CONTENT_CONTAINS); // Yes it is just XHTML
EXPECTED_XHTML_CONTENT_CONTAINS); // Yes it is just XHTML
}
@Test
public void pdfToXhtmlPdfBoxTest() throws Exception
{
transform(PDF_BOX, PDF, XHTML, MIMETYPE_PDF, MIMETYPE_XHTML, null,
EXPECTED_XHTML_CONTENT_CONTAINS);
EXPECTED_XHTML_CONTENT_CONTAINS);
}
@Test
public void pdfToHtmlPdfBoxTest() throws Exception
{
transform(PDF_BOX, PDF, HTML, MIMETYPE_PDF, MIMETYPE_HTML, null,
EXPECTED_XHTML_CONTENT_CONTAINS); // Yes it is just XHTML
EXPECTED_XHTML_CONTENT_CONTAINS); // Yes it is just XHTML
}
// --- Office ---
@@ -312,14 +314,14 @@ public class TikaTest extends AbstractBaseTest
public void msgToTxtOfficeTest() throws Exception
{
transform(OFFICE, MSG, TXT, MIMETYPE_OUTLOOK_MSG, MIMETYPE_TEXT_PLAIN, null,
EXPECTED_MSG_CONTENT_CONTAINS);
EXPECTED_MSG_CONTENT_CONTAINS);
}
@Test
public void docToTxtOfficeTest() throws Exception
{
transform(OFFICE, DOC, TXT, MIMETYPE_WORD, MIMETYPE_TEXT_PLAIN, null,
EXPECTED_TEXT_CONTENT_CONTAINS);
EXPECTED_TEXT_CONTENT_CONTAINS);
}
// --- Poi ---
@@ -328,7 +330,7 @@ public class TikaTest extends AbstractBaseTest
public void xslxToCsvPoiTest() throws Exception
{
transform(POI, XLSX, CSV, MIMETYPE_OPENXML_SPREADSHEET, MIMETYPE_TEXT_CSV, null,
EXPECTED_CSV_CONTENT_CONTAINS);
EXPECTED_CSV_CONTENT_CONTAINS);
}
// --- OOXML ---
@@ -337,14 +339,14 @@ public class TikaTest extends AbstractBaseTest
public void docxToTxtOoXmlTest() throws Exception
{
transform(OOXML, DOCX, TXT, MIMETYPE_OPENXML_WORDPROCESSING, MIMETYPE_TEXT_PLAIN, null,
EXPECTED_TEXT_CONTENT_CONTAINS);
EXPECTED_TEXT_CONTENT_CONTAINS);
}
@Test
public void pptxToTxtOoXmlTest() throws Exception
{
transform(OOXML, PPTX, TXT, MIMETYPE_OPENXML_PRESENTATION, MIMETYPE_TEXT_PLAIN, null,
EXPECTED_TEXT_CONTENT_CONTAINS);
EXPECTED_TEXT_CONTENT_CONTAINS);
}
// --- TikaAuto ---
@@ -353,14 +355,14 @@ public class TikaTest extends AbstractBaseTest
public void ppxtToTxtTikaAutoTest() throws Exception
{
transform(TIKA_AUTO, PPTX, TXT, MIMETYPE_OPENXML_PRESENTATION, MIMETYPE_TEXT_PLAIN, null,
EXPECTED_TEXT_CONTENT_CONTAINS);
EXPECTED_TEXT_CONTENT_CONTAINS);
}
@Test
public void doctToTxtTikaAutoTest() throws Exception
{
transform(TIKA_AUTO, DOCX, TXT, MIMETYPE_OPENXML_WORDPROCESSING, MIMETYPE_TEXT_PLAIN, null,
EXPECTED_TEXT_CONTENT_CONTAINS);
EXPECTED_TEXT_CONTENT_CONTAINS);
}
// --- TextMining ---
@@ -369,7 +371,7 @@ public class TikaTest extends AbstractBaseTest
public void docToTxtTextMiningTest() throws Exception
{
transform(TEXT_MINING, DOC, TXT, MIMETYPE_WORD, MIMETYPE_TEXT_PLAIN, null,
EXPECTED_TEXT_CONTENT_CONTAINS);
EXPECTED_TEXT_CONTENT_CONTAINS);
}
@Test
@@ -377,24 +379,22 @@ public class TikaTest extends AbstractBaseTest
{
mockTransformCommand(XLSX, XLSX, MIMETYPE_OPENXML_SPREADSHEET, false);
String metadata =
"{\"{http://www.alfresco.org/model/content/1.0}author\":\"author1\"," +
"\"{http://www.alfresco.org/model/content/1.0}title\":\"title1\"," +
"\"{http://www.alfresco.org/model/content/1.0}description\":[\"desc1\",\"desc2\"]," +
"\"{http://www.alfresco.org/model/content/1.0}created\":\"created1\"}";
String metadata = "{\"{http://www.alfresco.org/model/content/1.0}author\":\"author1\"," +
"\"{http://www.alfresco.org/model/content/1.0}title\":\"title1\"," +
"\"{http://www.alfresco.org/model/content/1.0}description\":[\"desc1\",\"desc2\"]," +
"\"{http://www.alfresco.org/model/content/1.0}created\":\"created1\"}";
MockHttpServletRequestBuilder requestBuilder =
super.mockMvcRequest(ENDPOINT_TRANSFORM, sourceFile,
"targetExtension", XLSX,
"metadata", metadata,
"targetMimetype", MIMETYPE_METADATA_EMBED,
"sourceMimetype", MIMETYPE_OPENXML_SPREADSHEET);
MockHttpServletRequestBuilder requestBuilder = super.mockMvcRequest(ENDPOINT_TRANSFORM, sourceFile,
"targetExtension", XLSX,
"metadata", metadata,
"targetMimetype", MIMETYPE_METADATA_EMBED,
"sourceMimetype", MIMETYPE_OPENXML_SPREADSHEET);
MvcResult result = mockMvc.perform(requestBuilder)
.andExpect(status().is(OK.value()))
.andExpect(header().string("Content-Disposition",
"attachment; filename*=UTF-8''transform." + targetExtension)).
andReturn();
"attachment; filename*=UTF-8''transform." + targetExtension))
.andReturn();
byte[] bytes = result.getResponse().getContentAsByteArray();
ByteArrayInputStream inputStream = new ByteArrayInputStream(bytes);
@@ -414,11 +414,11 @@ public class TikaTest extends AbstractBaseTest
{
mockTransformCommand(PDF, TXT, MIMETYPE_PDF, true);
mockMvc.perform(
mockMvcRequest(ENDPOINT_TRANSFORM, sourceFile, "targetExtension", targetExtension).param(
NOT_EXTRACT_BOOKMARKS_TEXT, "true"))
.andExpect(status().is(OK.value()))
.andExpect(header().string("Content-Disposition",
"attachment; filename*=UTF-8''transform." + targetExtension));
mockMvcRequest(ENDPOINT_TRANSFORM, sourceFile, "targetExtension", targetExtension).param(
NOT_EXTRACT_BOOKMARKS_TEXT, "true"))
.andExpect(status().is(OK.value()))
.andExpect(header().string("Content-Disposition",
"attachment; filename*=UTF-8''transform." + targetExtension));
}
@Override
@@ -445,11 +445,11 @@ public class TikaTest extends AbstractBaseTest
HttpHeaders headers = new HttpHeaders();
headers.set(CONTENT_DISPOSITION, "attachment; filename=quick." + sourceExtension);
ResponseEntity<Resource> response = new ResponseEntity<>(new FileSystemResource(
sourceFile), headers, OK);
sourceFile), headers, OK);
when(sharedFileStoreClient.retrieveFile(sourceFileRef)).thenReturn(response);
when(sharedFileStoreClient.saveFile(any()))
.thenReturn(new FileRefResponse(new FileRefEntity(targetFileRef)));
.thenReturn(new FileRefResponse(new FileRefEntity(targetFileRef)));
when(mockExecutionResult.getExitValue()).thenReturn(0);
// Update the Transformation Request with any specific params before sending it
@@ -458,16 +458,16 @@ public class TikaTest extends AbstractBaseTest
// Serialize and call the transformer
String tr = objectMapper.writeValueAsString(transformRequest);
String transformationReplyAsString = mockMvc
.perform(MockMvcRequestBuilders
.post(ENDPOINT_TRANSFORM)
.header(ACCEPT, APPLICATION_JSON_VALUE)
.header(CONTENT_TYPE, APPLICATION_JSON_VALUE)
.content(tr))
.andExpect(status().is(CREATED.value()))
.andReturn().getResponse().getContentAsString();
.perform(MockMvcRequestBuilders
.post(ENDPOINT_TRANSFORM)
.header(ACCEPT, APPLICATION_JSON_VALUE)
.header(CONTENT_TYPE, APPLICATION_JSON_VALUE)
.content(tr))
.andExpect(status().is(CREATED.value()))
.andReturn().getResponse().getContentAsString();
TransformReply transformReply = objectMapper.readValue(transformationReplyAsString,
TransformReply.class);
TransformReply.class);
// Assert the reply
assertEquals(transformRequest.getRequestId(), transformReply.getRequestId());
@@ -492,6 +492,6 @@ public class TikaTest extends AbstractBaseTest
"extractMapping",
"notExtractBookmarksText",
"metadata"),
getOptionNames(controller.transformConfig(0).getBody().getTransformOptions()));
getOptionNames(controller.transformConfig(0).getBody().getTransformOptions()));
}
}