inteligr8/alfresco-transform-core
1
0
Fork 0
mirror of https://github.com/Alfresco/alfresco-transform-core.git synced 2025-07-31 17:38:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix/mnt 25089 html transformations with ootb aio create extra whitespace (#1079)

Browse Source
This commit is contained in:
KushalBanik
2025-06-03 13:23:33 +05:30
committed by GitHub
parent 0c534f1081
commit cb9d070c9c
14 changed files with 1496 additions and 1324 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
docs/external-engine-configuration.md
View File

@@ -40,6 +40,7 @@ The following externalized T-engines properties are available:
| FILE_STORE_URL | T-Engine Port. | http://localhost:8099/alfresco/api/-default-/private/sfs/versions/1/file | | FILE_STORE_URL | T-Engine Port. | http://localhost:8099/alfresco/api/-default-/private/sfs/versions/1/file |
| TRANSFORM_ENGINE_REQUEST_QUEUE | T-Engine queue used for async requests. | org.alfresco.transform.engine.misc.acs | | TRANSFORM_ENGINE_REQUEST_QUEUE | T-Engine queue used for async requests. | org.alfresco.transform.engine.misc.acs |
| MISC_PDFBOX_DEFAULT_FONT | Default font used by PdfBox | NotoSans-Regular | | MISC_PDFBOX_DEFAULT_FONT | Default font used by PdfBox | NotoSans-Regular |
| MISC_HTML_COLLAPSE | Html Collasping Option for HTML to TXT transformation | true |
## Libreoffice ## Libreoffice
| Property | Description | Default value | | Property | Description | Default value |
@@ -98,4 +99,5 @@ The following externalized T-engines properties are available:
| IMAGEMAGICK_EXE | Path to Imagemagick EXE. | /usr/bin/convert | | IMAGEMAGICK_EXE | Path to Imagemagick EXE. | /usr/bin/convert |
| IMAGEMAGICK_CODERS | Path to Imagemagick custom coders. | | | IMAGEMAGICK_CODERS | Path to Imagemagick custom coders. | |
| IMAGEMAGICK_CONFIG | Path to Imagemagick custom config. | | | IMAGEMAGICK_CONFIG | Path to Imagemagick custom config. | |
| MISC_PDFBOX_DEFAULT_FONT | Default font used by PdfBox | NotoSans-Regular | | MISC_PDFBOX_DEFAULT_FONT | Default font used by PdfBox | NotoSans-Regular |
| MISC_HTML_COLLAPSE | Html Collasping Option for HTML to TXT transformation explicitly for Misc Engine | true |

4
engines/aio/src/main/resources/application-default.yaml
View File

@@ -26,4 +26,6 @@ transform:
unixOS: 'env FOO=#{"$"}{OUTPUT} exiftool -args -G1 -sep "|||" #{"$"}{INPUT}' unixOS: 'env FOO=#{"$"}{OUTPUT} exiftool -args -G1 -sep "|||" #{"$"}{INPUT}'
misc: misc:
pdfBox: pdfBox:
defaultFont: ${MISC_PDFBOX_DEFAULT_FONT:NotoSans-Regular} defaultFont: ${MISC_PDFBOX_DEFAULT_FONT:NotoSans-Regular}
htmlOptions:
collapseHtml: ${MISC_HTML_COLLAPSE:true}

254
engines/aio/src/test/java/org/alfresco/transform/aio/AIOTest.java
View File

@@ -1,126 +1,128 @@
/* /*
* #%L * #%L
* Alfresco Transform Core * Alfresco Transform Core
* %% * %%
* Copyright (C) 2005 - 2022 Alfresco Software Limited * Copyright (C) 2005 - 2025 Alfresco Software Limited
* %% * %%
* This file is part of the Alfresco software. * This file is part of the Alfresco software.
* - * -
* If the software was purchased under a paid Alfresco license, the terms of * If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is * the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms: * provided under the following open source license terms:
* - * -
* Alfresco is free software: you can redistribute it and/or modify * Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by * it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or * the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version. * (at your option) any later version.
* - * -
* Alfresco is distributed in the hope that it will be useful, * Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of * but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details. * GNU Lesser General Public License for more details.
* - * -
* You should have received a copy of the GNU Lesser General Public License * You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>. * along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L% * #L%
*/ */
package org.alfresco.transform.aio; package org.alfresco.transform.aio;
import org.alfresco.transform.base.AbstractBaseTest; import static org.junit.jupiter.api.Assertions.assertEquals;
import org.alfresco.transform.base.TransformController; import static org.junit.jupiter.api.Assertions.assertNotNull;
import org.alfresco.transform.config.TransformConfig; import static org.junit.jupiter.api.Assertions.assertNull;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import static org.alfresco.transform.base.TransformControllerTest.getLogMessagesFor;
import org.springframework.beans.factory.annotation.Autowired; import static org.alfresco.transform.common.Mimetype.MIMETYPE_HTML;
import org.springframework.http.ResponseEntity; import static org.alfresco.transform.common.Mimetype.MIMETYPE_TEXT_PLAIN;
import org.springframework.mock.web.MockMultipartFile; import static org.alfresco.transform.common.RequestParamMap.*;
import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder;
import java.nio.file.Files;
import java.nio.file.Files; import java.util.StringJoiner;
import java.util.StringJoiner;
import org.junit.jupiter.api.BeforeEach;
import static org.alfresco.transform.base.TransformControllerTest.getLogMessagesFor; import org.junit.jupiter.api.Test;
import static org.alfresco.transform.common.Mimetype.MIMETYPE_HTML; import org.springframework.beans.factory.annotation.Autowired;
import static org.alfresco.transform.common.Mimetype.MIMETYPE_TEXT_PLAIN; import org.springframework.http.ResponseEntity;
import static org.alfresco.transform.common.RequestParamMap.CONFIG_VERSION_DEFAULT; import org.springframework.mock.web.MockMultipartFile;
import static org.alfresco.transform.common.RequestParamMap.CONFIG_VERSION_LATEST; import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull; import org.alfresco.transform.base.AbstractBaseTest;
import static org.junit.jupiter.api.Assertions.assertNull; import org.alfresco.transform.base.TransformController;
import org.alfresco.transform.config.TransformConfig;
/**
* Test All-In-One. /**
*/ * Test All-In-One
public class AIOTest extends AbstractBaseTest */
{ public class AIOTest extends AbstractBaseTest
@Autowired {
private String coreVersion; @Autowired
private String coreVersion;
@BeforeEach
public void before() throws Exception @BeforeEach
{ public void before() throws Exception
sourceMimetype = MIMETYPE_HTML; {
targetMimetype = MIMETYPE_TEXT_PLAIN; sourceMimetype = MIMETYPE_HTML;
sourceExtension = "html"; targetMimetype = MIMETYPE_TEXT_PLAIN;
targetExtension = "txt"; sourceExtension = "html";
expectedOptions = null; targetExtension = "txt";
expectedSourceSuffix = null; expectedOptions = null;
sourceFileBytes = readTestFile(sourceExtension); expectedSourceSuffix = null;
expectedTargetFileBytes = Files.readAllBytes(getTestFile("quick2." + targetExtension, true).toPath()); sourceFileBytes = readTestFile(sourceExtension);
sourceFile = new MockMultipartFile("file", "quick." + sourceExtension, sourceMimetype, sourceFileBytes); expectedTargetFileBytes = Files.readAllBytes(getTestFile("quick3." + targetExtension, true).toPath());
} sourceFile = new MockMultipartFile("file", "quick." + sourceExtension, sourceMimetype, sourceFileBytes);
}
@Override
// Add extra required parameters to the request. @Override
protected MockHttpServletRequestBuilder mockMvcRequest(String url, MockMultipartFile sourceFile, String... params) // Add extra required parameters to the request.
{ protected MockHttpServletRequestBuilder mockMvcRequest(String url, MockMultipartFile sourceFile, String... params)
return super.mockMvcRequest(url, sourceFile, params) {
.param("targetMimetype", targetMimetype) return super.mockMvcRequest(url, sourceFile, params)
.param("sourceMimetype", sourceMimetype); .param("targetMimetype", targetMimetype)
} .param("sourceMimetype", sourceMimetype)
.param(HTML_COLLAPSE, "true");
@Test }
public void coreVersionNotSetInOriginalConfigTest()
{ @Test
ResponseEntity<TransformConfig> responseEntity = controller.transformConfig(Integer.valueOf(CONFIG_VERSION_DEFAULT)); public void coreVersionNotSetInOriginalConfigTest()
responseEntity.getBody().getTransformers().forEach(transformer -> { {
assertNull(transformer.getCoreVersion(), transformer.getTransformerName() + ResponseEntity<TransformConfig> responseEntity = controller.transformConfig(Integer.valueOf(CONFIG_VERSION_DEFAULT));
" should have had a null coreValue but was " + transformer.getCoreVersion()); responseEntity.getBody().getTransformers().forEach(transformer -> {
}); assertNull(transformer.getCoreVersion(), transformer.getTransformerName() +
} " should have had a null coreValue but was " + transformer.getCoreVersion());
});
@Test }
public void coreVersionSetInLatestConfigTest()
{ @Test
ResponseEntity<TransformConfig> responseEntity = controller.transformConfig(CONFIG_VERSION_LATEST); public void coreVersionSetInLatestConfigTest()
responseEntity.getBody().getTransformers().forEach(transformer -> { {
assertNotNull(transformer.getCoreVersion(), transformer.getTransformerName() + ResponseEntity<TransformConfig> responseEntity = controller.transformConfig(CONFIG_VERSION_LATEST);
" should have had a coreValue but was null. Should have been " + coreVersion); responseEntity.getBody().getTransformers().forEach(transformer -> {
}); assertNotNull(transformer.getCoreVersion(), transformer.getTransformerName() +
} " should have had a coreValue but was null. Should have been " + coreVersion);
});
@Test }
public void testStartupLogsIncludeEngineMessages()
{ @Test
StringJoiner controllerLogMessages = getLogMessagesFor(TransformController.class); public void testStartupLogsIncludeEngineMessages()
{
controller.startup(); StringJoiner controllerLogMessages = getLogMessagesFor(TransformController.class);
assertEquals( controller.startup();
"--------------------------------------------------------------------------------------------------------------------------------------------------------------\n"
+ "If the Alfresco software was purchased under a paid Alfresco license, the terms of the paid license agreement \n" assertEquals(
+ "will prevail. Otherwise, the software is provided under terms of the GNU LGPL v3 license. \n" "--------------------------------------------------------------------------------------------------------------------------------------------------------------\n"
+ "See the license at http://www.gnu.org/licenses/lgpl-3.0.txt. or in /LICENSE.txt \n" + "If the Alfresco software was purchased under a paid Alfresco license, the terms of the paid license agreement \n"
+ "\n" + "will prevail. Otherwise, the software is provided under terms of the GNU LGPL v3 license. \n"
+ "This transformer uses ImageMagick from ImageMagick Studio LLC. See the license at http://www.imagemagick.org/script/license.php or in /ImageMagick-license.txt\n" + "See the license at http://www.gnu.org/licenses/lgpl-3.0.txt. or in /LICENSE.txt \n"
+ "This transformer uses LibreOffice from The Document Foundation. See the license at https://www.libreoffice.org/download/license/ or in /libreoffice.txt\n" + "\n"
+ "This transformer uses libraries from Apache. See the license at http://www.apache.org/licenses/LICENSE-2.0. or in /Apache\\\\ 2.0.txt\n" + "This transformer uses ImageMagick from ImageMagick Studio LLC. See the license at http://www.imagemagick.org/script/license.php or in /ImageMagick-license.txt\n"
+ "This transformer uses htmlparser. See the license at http://htmlparser.sourceforge.net/license.html\n" + "This transformer uses LibreOffice from The Document Foundation. See the license at https://www.libreoffice.org/download/license/ or in /libreoffice.txt\n"
+ "This transformer uses alfresco-pdf-renderer which uses the PDFium library from Google Inc. See the license at https://pdfium.googlesource.com/pdfium/+/master/LICENSE or in /pdfium.txt\n" + "This transformer uses libraries from Apache. See the license at http://www.apache.org/licenses/LICENSE-2.0. or in /Apache\\\\ 2.0.txt\n"
+ "This transformer uses Tika from Apache. See the license at http://www.apache.org/licenses/LICENSE-2.0. or in /Apache\\ 2.0.txt\n" + "This transformer uses htmlparser. See the license at http://htmlparser.sourceforge.net/license.html\n"
+ "This transformer uses ExifTool by Phil Harvey. See license at https://exiftool.org/#license. or in /Perl-Artistic-License.txt\n" + "This transformer uses alfresco-pdf-renderer which uses the PDFium library from Google Inc. See the license at https://pdfium.googlesource.com/pdfium/+/master/LICENSE or in /pdfium.txt\n"
+ "--------------------------------------------------------------------------------------------------------------------------------------------------------------\n" + "This transformer uses Tika from Apache. See the license at http://www.apache.org/licenses/LICENSE-2.0. or in /Apache\\ 2.0.txt\n"
+ "Starting application components... Done", + "This transformer uses ExifTool by Phil Harvey. See license at https://exiftool.org/#license. or in /Perl-Artistic-License.txt\n"
controllerLogMessages.toString()); + "--------------------------------------------------------------------------------------------------------------------------------------------------------------\n"
} + "Starting application components... Done",
} controllerLogMessages.toString());
}
}

162
engines/aio/src/test/java/org/alfresco/transform/aio/AIOTikaTest.java
View File

@@ -1,80 +1,82 @@
/* /*
* #%L * #%L
* Alfresco Transform Core * Alfresco Transform Core
* %% * %%
* Copyright (C) 2005 - 2022 Alfresco Software Limited * Copyright (C) 2005 - 2025 Alfresco Software Limited
* %% * %%
* This file is part of the Alfresco software. * This file is part of the Alfresco software.
* - * -
* If the software was purchased under a paid Alfresco license", "the terms of * If the software was purchased under a paid Alfresco license", "the terms of
* the paid license agreement will prevail. Otherwise", "the software is * the paid license agreement will prevail. Otherwise", "the software is
* provided under the following open source license terms: * provided under the following open source license terms:
* - * -
* Alfresco is free software: you can redistribute it and/or modify * Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by * it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation", "either version 3 of the License", "or * the Free Software Foundation", "either version 3 of the License", "or
* (at your option) any later version. * (at your option) any later version.
* - * -
* Alfresco is distributed in the hope that it will be useful, * Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of * but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details. * GNU Lesser General Public License for more details.
* - * -
* You should have received a copy of the GNU Lesser General Public License * You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not", "see <http://www.gnu.org/licenses/>. * along with Alfresco. If not", "see <http://www.gnu.org/licenses/>.
* #L% * #L%
*/ */
package org.alfresco.transform.aio; package org.alfresco.transform.aio;
import com.google.common.collect.ImmutableSet; import static org.junit.jupiter.api.Assertions.assertEquals;
import org.alfresco.transform.tika.TikaTest;
import org.junit.jupiter.api.Test; import static org.alfresco.transform.base.html.OptionsHelper.getOptionNames;
import static org.alfresco.transform.base.html.OptionsHelper.getOptionNames; import com.google.common.collect.ImmutableSet;
import static org.junit.jupiter.api.Assertions.assertEquals; import org.junit.jupiter.api.Test;
/** import org.alfresco.transform.tika.TikaTest;
* Test Tika functionality in All-In-One.
*/ /**
public class AIOTikaTest extends TikaTest * Test Tika functionality in All-In-One.
{ */
@Test public class AIOTikaTest extends TikaTest
public void optionListTest() {
{ @Test
assertEquals(ImmutableSet.of( public void optionListTest()
"allowEnlargement", {
"allowPdfEnlargement", assertEquals(ImmutableSet.of(
"alphaRemove", "allowEnlargement",
"autoOrient", "allowPdfEnlargement",
"commandOptions", "alphaRemove",
"cropGravity", "autoOrient",
"cropHeight", "commandOptions",
"cropPercentage", "cropGravity",
"cropWidth", "cropHeight",
"cropXOffset", "cropPercentage",
"cropYOffset", "cropWidth",
"endPage", "cropXOffset",
"extractMapping", "cropYOffset",
"height", "endPage",
"includeContents", "extractMapping",
"maintainAspectRatio", "height",
"maintainPdfAspectRatio", "includeContents",
"metadata", "maintainAspectRatio",
"notExtractBookmarksText", "maintainPdfAspectRatio",
"page", "metadata",
"pageLimit", "notExtractBookmarksText",
"pdfFormat", "page",
"pdfOrientation", "pageLimit",
"resizeHeight", "pdfFormat",
"resizePercentage", "pdfOrientation",
"resizeWidth", "resizeHeight",
"startPage", "resizePercentage",
"targetEncoding", "resizeWidth",
"thumbnail", "startPage",
"width", "targetEncoding",
"pdfFont", "thumbnail",
"pdfFontSize" "width",
), "pdfFont",
getOptionNames(controller.transformConfig(0).getBody().getTransformOptions())); "pdfFontSize",
} "collapseHtml"),
} getOptionNames(controller.transformConfig(0).getBody().getTransformOptions()));
}
}

25
engines/misc/src/main/java/org/alfresco/transform/misc/MiscTransformEngine.java
View File

@@ -2,7 +2,7 @@
* #%L * #%L
* Alfresco Transform Core * Alfresco Transform Core
* %% * %%
* Copyright (C) 2005 - 2022 Alfresco Software Limited * Copyright (C) 2005 - 2025 Alfresco Software Limited
* %% * %%
* This file is part of the Alfresco software. * This file is part of the Alfresco software.
* - * -
@@ -26,21 +26,22 @@
*/ */
package org.alfresco.transform.misc; package org.alfresco.transform.misc;
import com.google.common.collect.ImmutableMap;
import org.alfresco.transform.base.TransformEngine;
import org.alfresco.transform.base.probes.ProbeTransform;
import org.alfresco.transform.config.reader.TransformConfigResourceReader;
import org.alfresco.transform.config.TransformConfig;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.Map;
import static org.alfresco.transform.base.logging.StandardMessages.COMMUNITY_LICENCE; import static org.alfresco.transform.base.logging.StandardMessages.COMMUNITY_LICENCE;
import static org.alfresco.transform.common.Mimetype.MIMETYPE_HTML; import static org.alfresco.transform.common.Mimetype.MIMETYPE_HTML;
import static org.alfresco.transform.common.Mimetype.MIMETYPE_TEXT_PLAIN; import static org.alfresco.transform.common.Mimetype.MIMETYPE_TEXT_PLAIN;
import static org.alfresco.transform.common.RequestParamMap.SOURCE_ENCODING; import static org.alfresco.transform.common.RequestParamMap.SOURCE_ENCODING;
import java.util.Map;
import com.google.common.collect.ImmutableMap;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.alfresco.transform.base.TransformEngine;
import org.alfresco.transform.base.probes.ProbeTransform;
import org.alfresco.transform.config.TransformConfig;
import org.alfresco.transform.config.reader.TransformConfigResourceReader;
@Component @Component
public class MiscTransformEngine implements TransformEngine public class MiscTransformEngine implements TransformEngine
{ {
@@ -74,6 +75,6 @@ public class MiscTransformEngine implements TransformEngine
public ProbeTransform getProbeTransform() public ProbeTransform getProbeTransform()
{ {
return new ProbeTransform("probe.html", MIMETYPE_HTML, MIMETYPE_TEXT_PLAIN, transformOptions, return new ProbeTransform("probe.html", MIMETYPE_HTML, MIMETYPE_TEXT_PLAIN, transformOptions,
119, 30, 150, 1024, 60 * 2 + 1, 60 * 2); 107, 30, 150, 1024, 60 * 2 + 1, 60 * 2);
} }
} }

418
engines/misc/src/main/java/org/alfresco/transform/misc/transformers/HtmlParserContentTransformer.java
View File

@@ -1,203 +1,215 @@
/* /*
* #%L * #%L
* Alfresco Transform Core * Alfresco Transform Core
* %% * %%
* Copyright (C) 2005 - 2022 Alfresco Software Limited * Copyright (C) 2005 - 2025 Alfresco Software Limited
* %% * %%
* This file is part of the Alfresco software. * This file is part of the Alfresco software.
* - * -
* If the software was purchased under a paid Alfresco license, the terms of * If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is * the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms: * provided under the following open source license terms:
* - * -
* Alfresco is free software: you can redistribute it and/or modify * Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by * it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or * the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version. * (at your option) any later version.
* - * -
* Alfresco is distributed in the hope that it will be useful, * Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of * but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details. * GNU Lesser General Public License for more details.
* - * -
* You should have received a copy of the GNU Lesser General Public License * You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>. * along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L% * #L%
*/ */
package org.alfresco.transform.misc.transformers; package org.alfresco.transform.misc.transformers;
import org.alfresco.transform.base.TransformManager; import static org.alfresco.transform.common.RequestParamMap.HTML_COLLAPSE;
import org.alfresco.transform.base.util.CustomTransformerFileAdaptor; import static org.alfresco.transform.common.RequestParamMap.SOURCE_ENCODING;
import org.htmlparser.Parser;
import org.htmlparser.beans.StringBean; import java.io.BufferedWriter;
import org.htmlparser.util.ParserException; import java.io.File;
import org.slf4j.Logger; import java.io.FileOutputStream;
import org.slf4j.LoggerFactory; import java.io.OutputStreamWriter;
import org.springframework.stereotype.Component; import java.io.Writer;
import java.net.URLConnection;
import java.io.BufferedWriter; import java.nio.charset.Charset;
import java.io.File; import java.nio.charset.IllegalCharsetNameException;
import java.io.FileOutputStream; import java.util.Map;
import java.io.OutputStreamWriter;
import java.io.Writer; import org.htmlparser.Parser;
import java.net.URLConnection; import org.htmlparser.beans.StringBean;
import java.nio.charset.Charset; import org.htmlparser.util.ParserException;
import java.nio.charset.IllegalCharsetNameException; import org.slf4j.Logger;
import java.util.Map; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import static org.alfresco.transform.common.RequestParamMap.SOURCE_ENCODING; import org.springframework.stereotype.Component;
/** import org.alfresco.transform.base.TransformManager;
* Content transformer which wraps the HTML Parser library for import org.alfresco.transform.base.util.CustomTransformerFileAdaptor;
* parsing HTML content.
* /**
* <p> * Content transformer which wraps the HTML Parser library for parsing HTML content.
* This code is based on a class of the same name originally implemented in alfresco-repository. *
* </p> * <p>
* * This code is based on a class of the same name originally implemented in alfresco-repository.
* <p> * </p>
* Since HTML Parser was updated from v1.6 to v2.1, META tags *
* defining an encoding for the content via http-equiv=Content-Type * <p>
* will ONLY be respected if the encoding of the content item * Since HTML Parser was updated from v1.6 to v2.1, META tags defining an encoding for the content via http-equiv=Content-Type will ONLY be respected if the encoding of the content item itself is set to ISO-8859-1.
* itself is set to ISO-8859-1. * </p>
* </p> *
* * <p>
* <p> * Tika Note - could be converted to use the Tika HTML parser, but we'd potentially need a custom text handler to replicate the current settings around links and non-breaking spaces.
* Tika Note - could be converted to use the Tika HTML parser, * </p>
* but we'd potentially need a custom text handler to replicate *
* the current settings around links and non-breaking spaces. * @author Derek Hulley
* </p> * @author eknizat
* * @see <a href="http://htmlparser.sourceforge.net/">http://htmlparser.sourceforge.net</a>
* @author Derek Hulley * @see org.htmlparser.beans.StringBean
* @author eknizat * @see <a href="http://sourceforge.net/tracker/?func=detail&aid=1644504&group_id=24399&atid=381401">HTML Parser</a>
* @see <a href="http://htmlparser.sourceforge.net/">http://htmlparser.sourceforge.net</a> */
* @see org.htmlparser.beans.StringBean @Component
* @see <a href="http://sourceforge.net/tracker/?func=detail&aid=1644504&group_id=24399&atid=381401">HTML Parser</a> public class HtmlParserContentTransformer implements CustomTransformerFileAdaptor
*/ {
@Component private static final Logger logger = LoggerFactory.getLogger(
public class HtmlParserContentTransformer implements CustomTransformerFileAdaptor HtmlParserContentTransformer.class);
{
private static final Logger logger = LoggerFactory.getLogger( @Value("${transform.core.misc.htmlOptions.collapseHtml:true}")
HtmlParserContentTransformer.class); private String collapseOptionDefault;
@Override @Override
public String getTransformerName() public String getTransformerName()
{ {
return "html"; return "html";
} }
@Override @Override
public void transform(final String sourceMimetype, final String targetMimetype, public void transform(final String sourceMimetype, final String targetMimetype,
final Map<String, String> transformOptions, final Map<String, String> transformOptions,
final File sourceFile, final File targetFile, TransformManager transformManager) throws Exception final File sourceFile, final File targetFile, TransformManager transformManager) throws Exception
{ {
String sourceEncoding = transformOptions.get(SOURCE_ENCODING); String sourceEncoding = transformOptions.get(SOURCE_ENCODING);
checkEncodingParameter(sourceEncoding, SOURCE_ENCODING); checkEncodingParameter(sourceEncoding, SOURCE_ENCODING);
boolean collapse;
if (logger.isDebugEnabled())
{ var collapseOption = transformOptions.get(HTML_COLLAPSE);
logger.debug("Performing HTML to text transform with sourceEncoding=" + sourceEncoding); // If the collapse option is set, use it, otherwise use the default value
} if (collapseOption != null && (collapseOption.trim().equalsIgnoreCase("true") || collapseOption.trim().equalsIgnoreCase("false")))
{
// Create the extractor collapse = Boolean.parseBoolean(collapseOption);
EncodingAwareStringBean extractor = new EncodingAwareStringBean(); }
extractor.setCollapse(false); else
extractor.setLinks(false); {
extractor.setReplaceNonBreakingSpaces(false); // Use the default value from the configuration
extractor.setURL(sourceFile, sourceEncoding); collapse = collapseOptionDefault == null || Boolean.parseBoolean(collapseOptionDefault);
// get the text if (logger.isDebugEnabled())
String text = extractor.getStrings(); {
logger.debug("Using default html collapse option: " + collapseOptionDefault);
// write it to the writer }
try (Writer writer = new BufferedWriter( }
new OutputStreamWriter(new FileOutputStream(targetFile))))
{ if (logger.isDebugEnabled())
writer.write(text); {
} logger.debug("Performing HTML to text transform with sourceEncoding=" + sourceEncoding);
} }
private void checkEncodingParameter(String encoding, String parameterName) // Create the extractor
{ EncodingAwareStringBean extractor = new EncodingAwareStringBean();
try extractor.setCollapse(collapse);
{ extractor.setLinks(false);
if (encoding != null && !Charset.isSupported(encoding)) extractor.setReplaceNonBreakingSpaces(false);
{ extractor.setURL(sourceFile, sourceEncoding);
throw new IllegalArgumentException( // get the text
parameterName + "=" + encoding + " is not supported by the JVM."); String text = extractor.getStrings();
}
} // write it to the writer
catch (IllegalCharsetNameException e) try (Writer writer = new BufferedWriter(
{ new OutputStreamWriter(new FileOutputStream(targetFile))))
throw new IllegalArgumentException( {
parameterName + "=" + encoding + " is not a valid encoding."); writer.write(text);
} }
} }
/** private void checkEncodingParameter(String encoding, String parameterName)
* <p> {
* This code is based on a class of the same name, originally implemented in alfresco-repository. try
* </p> {
* if (encoding != null && !Charset.isSupported(encoding))
* A version of {@link StringBean} which allows control of the {
* encoding in the underlying HTML Parser. throw new IllegalArgumentException(
* Unfortunately, StringBean doesn't allow easy over-riding of parameterName + "=" + encoding + " is not supported by the JVM.");
* this, so we have to duplicate some code to control this. }
* This allows us to correctly handle HTML files where the encoding }
* is specified against the content property (rather than in the catch (IllegalCharsetNameException e)
* HTML Head Meta), see ALF-10466 for details. {
*/ throw new IllegalArgumentException(
public static class EncodingAwareStringBean extends StringBean parameterName + "=" + encoding + " is not a valid encoding.");
{ }
private static final long serialVersionUID = -9033414360428669553L; }
/** /**
* Sets the File to extract strings from, and the encoding * <p>
* it's in (if known to Alfresco) * This code is based on a class of the same name, originally implemented in alfresco-repository.
* * </p>
* @param file The File that text should be fetched from. *
* @param encoding The encoding of the input * A version of {@link StringBean} which allows control of the encoding in the underlying HTML Parser. Unfortunately, StringBean doesn't allow easy over-riding of this, so we have to duplicate some code to control this. This allows us to correctly handle HTML files where the encoding is specified against the content property (rather than in the HTML Head Meta), see ALF-10466 for details.
*/ */
public void setURL(File file, String encoding) public static class EncodingAwareStringBean extends StringBean
{ {
String previousURL = getURL(); private static final long serialVersionUID = -9033414360428669553L;
String newURL = file.getAbsolutePath();
/**
if (previousURL == null || !newURL.equals(previousURL)) * Sets the File to extract strings from, and the encoding it's in (if known to Alfresco)
{ *
try * @param file
{ * The File that text should be fetched from.
URLConnection conn = getConnection(); * @param encoding
* The encoding of the input
if (null == mParser) */
{ public void setURL(File file, String encoding)
mParser = new Parser(newURL); {
} String previousURL = getURL();
else String newURL = file.getAbsolutePath();
{
mParser.setURL(newURL); if (previousURL == null || !newURL.equals(previousURL))
} {
try
if (encoding != null) {
{ URLConnection conn = getConnection();
mParser.setEncoding(encoding);
} if (null == mParser)
{
mPropertySupport.firePropertyChange(StringBean.PROP_URL_PROPERTY, previousURL, mParser = new Parser(newURL);
getURL()); }
mPropertySupport.firePropertyChange(StringBean.PROP_CONNECTION_PROPERTY, conn, else
mParser.getConnection()); {
setStrings(); mParser.setURL(newURL);
} }
catch (ParserException pe)
{ if (encoding != null)
updateStrings(pe.toString()); {
} mParser.setEncoding(encoding);
} }
}
mPropertySupport.firePropertyChange(StringBean.PROP_URL_PROPERTY, previousURL,
public String getEncoding() getURL());
{ mPropertySupport.firePropertyChange(StringBean.PROP_CONNECTION_PROPERTY, conn,
return mParser.getEncoding(); mParser.getConnection());
} setStrings();
} }
} catch (ParserException pe)
{
updateStrings(pe.toString());
}
}
}
public String getEncoding()
{
return mParser.getEncoding();
}
}
}

4
engines/misc/src/main/resources/application-default.yaml
View File

@@ -4,4 +4,6 @@ transform:
core: core:
misc: misc:
pdfBox: pdfBox:
defaultFont: ${MISC_PDFBOX_DEFAULT_FONT:NotoSans-Regular} defaultFont: ${MISC_PDFBOX_DEFAULT_FONT:NotoSans-Regular}
htmlOptions:
collapseHtml: ${MISC_HTML_COLLAPSE:true}

6
engines/misc/src/main/resources/misc_engine_config.json
View File

@@ -1,5 +1,8 @@
{ {
"transformOptions": { "transformOptions": {
"htmlOptions": [
{"value": {"name": "collapseHtml"}}
],
"textToPdfOptions": [ "textToPdfOptions": [
{"value": {"name": "pageLimit"}}, {"value": {"name": "pageLimit"}},
{"value": {"name": "pdfFont"}}, {"value": {"name": "pdfFont"}},
@@ -24,8 +27,7 @@
"supportedSourceAndTargetList": [ "supportedSourceAndTargetList": [
{"sourceMediaType": "text/html", "targetMediaType": "text/plain"} {"sourceMediaType": "text/html", "targetMediaType": "text/plain"}
], ],
"transformOptions": [ "transformOptions": ["htmlOptions"]
]
}, },
{ {
"transformerName": "string", "transformerName": "string",

1002
engines/misc/src/test/java/org/alfresco/transform/misc/MiscTest.java
View File

File diff suppressed because it is too large Load Diff

462
engines/misc/src/test/java/org/alfresco/transform/misc/transformers/HtmlParserContentTransformerTest.java
View File

@@ -1,162 +1,300 @@
/* /*
* #%L * #%L
* Alfresco Transform Core * Alfresco Transform Core
* %% * %%
* Copyright (C) 2005 - 2022 Alfresco Software Limited * Copyright (C) 2005 - 2025 Alfresco Software Limited
* %% * %%
* This file is part of the Alfresco software. * This file is part of the Alfresco software.
* - * -
* If the software was purchased under a paid Alfresco license, the terms of * If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is * the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms: * provided under the following open source license terms:
* - * -
* Alfresco is free software: you can redistribute it and/or modify * Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by * it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or * the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version. * (at your option) any later version.
* - * -
* Alfresco is distributed in the hope that it will be useful, * Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of * but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details. * GNU Lesser General Public License for more details.
* - * -
* You should have received a copy of the GNU Lesser General Public License * You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>. * along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L% * #L%
*/ */
package org.alfresco.transform.misc.transformers; package org.alfresco.transform.misc.transformers;
import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.fail;
import java.io.File;
import java.io.FileOutputStream; import static org.alfresco.transform.common.RequestParamMap.HTML_COLLAPSE;
import java.io.OutputStreamWriter; import static org.alfresco.transform.common.RequestParamMap.SOURCE_ENCODING;
import java.nio.file.Files;
import java.util.HashMap; import java.io.File;
import java.util.Map; import java.io.FileOutputStream;
import java.io.OutputStreamWriter;
import static org.alfresco.transform.common.RequestParamMap.SOURCE_ENCODING; import java.nio.file.Files;
import static org.junit.jupiter.api.Assertions.assertEquals; import java.util.HashMap;
import java.util.Map;
public class HtmlParserContentTransformerTest
{ import org.junit.jupiter.api.Test;
private static final String SOURCE_MIMETYPE = "text/html"; import org.junit.jupiter.params.ParameterizedTest;
private static final String TARGET_MIMETYPE = "text/plain"; import org.junit.jupiter.params.provider.ValueSource;
HtmlParserContentTransformer transformer = new HtmlParserContentTransformer(); public class HtmlParserContentTransformerTest
{
/** private static final String SOURCE_MIMETYPE = "text/html";
* Checks that we correctly handle text in different encodings, private static final String TARGET_MIMETYPE = "text/plain";
* no matter if the encoding is specified on the Content Property
* or in a meta tag within the HTML itself. (ALF-10466) /**
* * Checks that we correctly handle text in different encodings, no matter if the encoding is specified on the Content Property or in a meta tag within the HTML itself. (ALF-10466)
* On Windows, org.htmlparser.beans.StringBean.carriageReturn() appends a new system dependent new line *
* so we must be careful when checking the returned text * On Windows, org.htmlparser.beans.StringBean.carriageReturn() appends a new system dependent new line so we must be careful when checking the returned text
*/ */
@Test @Test
public void testEncodingHandling() throws Exception public void testEncodingHandling() throws Exception
{ {
final String NEWLINE = System.getProperty("line.separator"); final HtmlParserContentTransformer transformer = new HtmlParserContentTransformer();
final String TITLE = "Testing!"; final String newline = System.getProperty("line.separator");
final String TEXT_P1 = "This is some text in English"; final String title = "Testing!";
final String TEXT_P2 = "This is more text in English"; final String textp1 = "This is some text in English";
final String TEXT_P3 = "C'est en Fran\u00e7ais et Espa\u00f1ol"; final String textp2 = "This is more text in English";
String partA = "<html><head><title>" + TITLE + "</title></head>" + NEWLINE; final String textp3 = "C'est en Fran\u00e7ais et Espa\u00f1ol";
String partB = "<body><p>" + TEXT_P1 + "</p>" + NEWLINE + String partA = "<html><head><title>" + title + "</title></head>" + newline;
"<p>" + TEXT_P2 + "</p>" + NEWLINE + String partB = "<body><p>" + textp1 + "</p>" + newline +
"<p>" + TEXT_P3 + "</p>" + NEWLINE; "<p>" + textp2 + "</p>" + newline +
String partC = "</body></html>"; "<p>" + textp3 + "</p>" + newline;
final String expected = TITLE + NEWLINE + TEXT_P1 + NEWLINE + TEXT_P2 + NEWLINE + TEXT_P3 + NEWLINE; String partC = "</body></html>";
final String expected = title + newline + textp1 + newline + textp2 + newline + textp3;
File tmpS = null;
File tmpD = null; File tmpS = null;
File tmpD = null;
try
{ try
// Content set to ISO 8859-1 {
tmpS = File.createTempFile("AlfrescoTestSource_", ".html"); // Content set to ISO 8859-1
writeToFile(tmpS, partA + partB + partC, "ISO-8859-1"); tmpS = File.createTempFile("AlfrescoTestSource_", ".html");
writeToFile(tmpS, partA + partB + partC, "ISO-8859-1");
tmpD = File.createTempFile("AlfrescoTestTarget_", ".txt");
tmpD = File.createTempFile("AlfrescoTestTarget_", ".txt");
Map<String, String> parameters = new HashMap<>();
parameters.put(SOURCE_ENCODING, "ISO-8859-1"); Map<String, String> parameters = new HashMap<>();
transformer.transform(SOURCE_MIMETYPE, TARGET_MIMETYPE, parameters, tmpS, tmpD, null); parameters.put(SOURCE_ENCODING, "ISO-8859-1");
parameters.put(HTML_COLLAPSE, String.valueOf(true));
assertEquals(expected, readFromFile(tmpD, "UTF-8")); transformer.transform(SOURCE_MIMETYPE, TARGET_MIMETYPE, parameters, tmpS, tmpD, null);
tmpS.delete();
tmpD.delete(); assertEquals(expected, readFromFile(tmpD, "UTF-8"));
tmpS.delete();
// Content set to UTF-8 tmpD.delete();
tmpS = File.createTempFile("AlfrescoTestSource_", ".html");
writeToFile(tmpS, partA + partB + partC, "UTF-8"); // Content set to UTF-8
tmpS = File.createTempFile("AlfrescoTestSource_", ".html");
tmpD = File.createTempFile("AlfrescoTestTarget_", ".txt"); writeToFile(tmpS, partA + partB + partC, "UTF-8");
parameters = new HashMap<>();
parameters.put(SOURCE_ENCODING, "UTF-8"); tmpD = File.createTempFile("AlfrescoTestTarget_", ".txt");
transformer.transform(SOURCE_MIMETYPE, TARGET_MIMETYPE, parameters, tmpS, tmpD, null); parameters = new HashMap<>();
assertEquals(expected, readFromFile(tmpD, "UTF-8")); parameters.put(SOURCE_ENCODING, "UTF-8");
tmpS.delete(); parameters.put(HTML_COLLAPSE, String.valueOf(true));
tmpD.delete(); transformer.transform(SOURCE_MIMETYPE, TARGET_MIMETYPE, parameters, tmpS, tmpD, null);
assertEquals(expected, readFromFile(tmpD, "UTF-8"));
// Content set to UTF-16 tmpS.delete();
tmpS = File.createTempFile("AlfrescoTestSource_", ".html"); tmpD.delete();
writeToFile(tmpS, partA + partB + partC, "UTF-16");
// Content set to UTF-16
tmpD = File.createTempFile("AlfrescoTestTarget_", ".txt"); tmpS = File.createTempFile("AlfrescoTestSource_", ".html");
parameters = new HashMap<>(); writeToFile(tmpS, partA + partB + partC, "UTF-16");
parameters.put(SOURCE_ENCODING, "UTF-16");
transformer.transform(SOURCE_MIMETYPE, TARGET_MIMETYPE, parameters, tmpS, tmpD, null); tmpD = File.createTempFile("AlfrescoTestTarget_", ".txt");
assertEquals(expected, readFromFile(tmpD, "UTF-8")); parameters = new HashMap<>();
tmpS.delete(); parameters.put(HTML_COLLAPSE, String.valueOf(true));
tmpD.delete(); parameters.put(SOURCE_ENCODING, "UTF-16");
transformer.transform(SOURCE_MIMETYPE, TARGET_MIMETYPE, parameters, tmpS, tmpD, null);
// Note - since HTML Parser 2.0 META tags specifying the assertEquals(expected, readFromFile(tmpD, "UTF-8"));
// document encoding will ONLY be respected if the original tmpS.delete();
// content type was set to ISO-8859-1. tmpD.delete();
//
// This means there is now only one test which we can perform // Note - since HTML Parser 2.0 META tags specifying the
// to ensure that this now-limited overriding of the encoding // document encoding will ONLY be respected if the original
// takes effect. // content type was set to ISO-8859-1.
//
// Content set to ISO 8859-1, meta set to UTF-8 // This means there is now only one test which we can perform
tmpS = File.createTempFile("AlfrescoTestSource_", ".html"); // to ensure that this now-limited overriding of the encoding
String str = partA + // takes effect.
"<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\">" +
partB + partC; // Content set to ISO 8859-1, meta set to UTF-8
tmpS = File.createTempFile("AlfrescoTestSource_", ".html");
writeToFile(tmpS, str, "UTF-8"); String str = partA +
"<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\">" +
tmpD = File.createTempFile("AlfrescoTestTarget_", ".txt"); partB + partC;
parameters = new HashMap<>(); writeToFile(tmpS, str, "UTF-8");
parameters.put(SOURCE_ENCODING, "ISO-8859-1");
transformer.transform(SOURCE_MIMETYPE, TARGET_MIMETYPE, parameters, tmpS, tmpD, null); tmpD = File.createTempFile("AlfrescoTestTarget_", ".txt");
assertEquals(expected, readFromFile(tmpD, "UTF-8"));
tmpS.delete(); parameters = new HashMap<>();
tmpD.delete(); parameters.put(SOURCE_ENCODING, "ISO-8859-1");
parameters.put(HTML_COLLAPSE, String.valueOf(true));
// Note - we can't test UTF-16 with only a meta encoding, transformer.transform(SOURCE_MIMETYPE, TARGET_MIMETYPE, parameters, tmpS, tmpD, null);
// because without that the parser won't know about the assertEquals(expected, readFromFile(tmpD, "UTF-8"));
// 2 byte format so won't be able to identify the meta tag tmpS.delete();
} tmpD.delete();
finally
{ // Note - we can't test UTF-16 with only a meta encoding,
if (tmpS != null && tmpS.exists()) tmpS.delete(); // because without that the parser won't know about the
if (tmpD != null && tmpD.exists()) tmpD.delete(); // 2 byte format so won't be able to identify the meta tag
} }
} catch (Exception e)
{
private void writeToFile(File file, String content, String encoding) throws Exception fail("Test Failed: " + e.getMessage()); // fail the test if any exception occurs
{ }
try (OutputStreamWriter ow = new OutputStreamWriter(new FileOutputStream(file), encoding)) finally
{ {
ow.append(content); if (tmpS != null && tmpS.exists())
} {
} tmpS.delete();
}
private String readFromFile(File file, final String encoding) throws Exception if (tmpD != null && tmpD.exists())
{ {
return new String(Files.readAllBytes(file.toPath()), encoding); tmpD.delete();
} }
} }
}
/**
* Tests the transformer with different collapsing methods. If the collapsing is set to false, it should not collapse the new lines between paragraphs. If the collapsing is set to true, it should collapse the new lines.
*/
@ParameterizedTest
@ValueSource(booleans = {true, false})
public void testTransformerWithDifferentCollapsingMethods(boolean shouldCollapse)
{
final HtmlParserContentTransformer transformer = new HtmlParserContentTransformer();
final String newline = System.getProperty("line.separator");
final String title = "Testing!";
final String textp1 = "This is some text in English";
final String textp2 = "This is more text in English";
final String textp3 = "C'est en Fran\u00e7ais et Espa\u00f1ol";
String partA = "<html><head><title>" + title + "</title></head>" + newline;
String partB = "<body><p>" + textp1 + "</p>" + newline +
"<p>" + textp2 + "</p>" + newline +
"<p>" + textp3 + "</p>" + newline;
String partC = "</body></html>";
final String expected = title + newline + textp1 + newline + textp2 + newline + textp3 + (shouldCollapse ? "" : newline); // Just a added newline if collapsing is not collapsing
File tmpS = null;
File tmpD = null;
try
{
tmpS = File.createTempFile("AlfrescoTestSource_", ".html");
writeToFile(tmpS, partA + partB + partC, "UTF-8");
tmpD = File.createTempFile("AlfrescoTestTarget_", ".txt");
Map<String, String> parameters = new HashMap<>();
parameters.put(SOURCE_ENCODING, "UTF-8");
parameters.put(HTML_COLLAPSE, String.valueOf(shouldCollapse));
transformer.transform(SOURCE_MIMETYPE, TARGET_MIMETYPE, parameters, tmpS, tmpD, null);
assertEquals(expected, readFromFile(tmpD, "UTF-8"));
tmpS.delete();
tmpD.delete();
}
catch (Exception e)
{
fail("Test Failed: " + e.getMessage()); // fail the test if any exception occurs
}
finally
{
if (tmpS != null && tmpS.exists())
{
tmpS.delete();
}
if (tmpD != null && tmpD.exists())
{
tmpD.delete();
}
}
}
/**
* Tests the transformer with wrong boolean values for the collapse option. It should not throw an exception and should use the default value for collapsing.
*/
@ParameterizedTest
@ValueSource(strings = {"cat", "dog", "", "1234abcd", "@#$%"})
public void testTransformerWithWrongBooleanValues(String booleanValues)
{
final HtmlParserContentTransformer transformer = new HtmlParserContentTransformer();
final String newline = System.getProperty("line.separator");
final String title = "Testing!";
final String textp1 = "This is some text in English";
final String textp2 = "This is more text in English";
final String textp3 = "C'est en Fran\u00e7ais et Espa\u00f1ol";
String partA = "<html><head><title>" + title + "</title></head>" + newline;
String partB = "<body><p>" + textp1 + "</p>" + newline +
"<p>" + textp2 + "</p>" + newline +
"<p>" + textp3 + "</p>" + newline;
String partC = "</body></html>";
final String expected = title + newline + textp1 + newline + textp2 + newline + textp3;
File tmpS = null;
File tmpD = null;
try
{
tmpS = File.createTempFile("AlfrescoTestSource_", ".html");
writeToFile(tmpS, partA + partB + partC, "UTF-8");
tmpD = File.createTempFile("AlfrescoTestTarget_", ".txt");
Map<String, String> parameters = new HashMap<>();
parameters.put(SOURCE_ENCODING, "UTF-8");
parameters.put(HTML_COLLAPSE, booleanValues);
transformer.transform(SOURCE_MIMETYPE, TARGET_MIMETYPE, parameters, tmpS, tmpD, null);
assertEquals(expected, readFromFile(tmpD, "UTF-8"));
tmpS.delete();
tmpD.delete();
}
catch (Exception e)
{
fail("Test Failed: " + e.getMessage()); // fail the test if any exception occurs
}
finally
{
if (tmpS != null && tmpS.exists())
{
tmpS.delete();
}
if (tmpD != null && tmpD.exists())
{
tmpD.delete();
}
}
}
private void writeToFile(File file, String content, String encoding)
{
try (OutputStreamWriter ow = new OutputStreamWriter(new FileOutputStream(file), encoding))
{
ow.append(content);
}
catch (Exception e)
{
fail("Failed to write to file: " + e.getMessage()); // fail the test if any exception occurs
}
}
private String readFromFile(File file, final String encoding)
{
try
{
return new String(Files.readAllBytes(file.toPath()), encoding);
}
catch (Exception e)
{
fail("Failed to read from file: " + e.getMessage());
return null; // Return null if there is an error reading the file
}
}
}

4
engines/misc/src/test/resources/misc_engine_config.json
View File

@@ -1,5 +1,8 @@
{ {
"transformOptions": { "transformOptions": {
"htmlOptions": [
{"value": {"name": "collapseHtml"}}
],
"textToPdfOptions": [ "textToPdfOptions": [
{"value": {"name": "pageLimit"}} {"value": {"name": "pageLimit"}}
], ],
@@ -17,6 +20,7 @@
{"sourceMediaType": "text/html", "targetMediaType": "text/plain"} {"sourceMediaType": "text/html", "targetMediaType": "text/plain"}
], ],
"transformOptions": [ "transformOptions": [
"htmlOptions"
] ]
}, },
{ {

2
engines/misc/src/test/resources/quick3.txt Normal file
View File

@@ -0,0 +1,2 @@
The quick brown fox jumps over the lazy dog
The quick brown fox jumps over the lazy dog

284
engines/tika/src/test/java/org/alfresco/transform/tika/TikaTest.java
View File

@@ -2,7 +2,7 @@
* #%L * #%L
* Alfresco Transform Core * Alfresco Transform Core
* %% * %%
* Copyright (C) 2005 - 2022 Alfresco Software Limited * Copyright (C) 2005 - 2025 Alfresco Software Limited
* %% * %%
* This file is part of the Alfresco software. * This file is part of the Alfresco software.
* - * -
@@ -26,31 +26,21 @@
*/ */
package org.alfresco.transform.tika; package org.alfresco.transform.tika;
import com.google.common.collect.ImmutableSet; import static org.junit.jupiter.api.Assertions.assertEquals;
import org.alfresco.transform.base.AbstractBaseTest; import static org.junit.jupiter.api.Assertions.assertTrue;
import org.alfresco.transform.base.executors.RuntimeExec; import static org.mockito.ArgumentMatchers.any;
import org.alfresco.transform.base.model.FileRefEntity; import static org.mockito.Mockito.when;
import org.alfresco.transform.base.model.FileRefResponse; import static org.springframework.http.HttpHeaders.ACCEPT;
import org.alfresco.transform.client.model.TransformReply; import static org.springframework.http.HttpHeaders.CONTENT_DISPOSITION;
import org.alfresco.transform.client.model.TransformRequest; import static org.springframework.http.HttpHeaders.CONTENT_TYPE;
import org.apache.poi.ooxml.POIXMLProperties; import static org.springframework.http.HttpStatus.CREATED;
import org.apache.poi.xssf.usermodel.XSSFWorkbook; import static org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR;
import org.junit.jupiter.api.BeforeEach; import static org.springframework.http.HttpStatus.OK;
import org.junit.jupiter.api.Test; import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE;
import org.mockito.Mock; import static org.springframework.http.MediaType.APPLICATION_PDF_VALUE;
import org.springframework.core.io.FileSystemResource; import static org.springframework.http.MediaType.TEXT_PLAIN_VALUE;
import org.springframework.core.io.Resource; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header;
import org.springframework.http.HttpHeaders; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import org.springframework.http.ResponseEntity;
import org.springframework.mock.web.MockMultipartFile;
import org.springframework.test.web.servlet.MvcResult;
import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder;
import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.util.UUID;
import static org.alfresco.transform.base.html.OptionsHelper.getOptionNames; import static org.alfresco.transform.base.html.OptionsHelper.getOptionNames;
import static org.alfresco.transform.common.Mimetype.MIMETYPE_HTML; import static org.alfresco.transform.common.Mimetype.MIMETYPE_HTML;
@@ -89,21 +79,33 @@ import static org.alfresco.transform.tika.transformers.Tika.XHTML;
import static org.alfresco.transform.tika.transformers.Tika.XLSX; import static org.alfresco.transform.tika.transformers.Tika.XLSX;
import static org.alfresco.transform.tika.transformers.Tika.XML; import static org.alfresco.transform.tika.transformers.Tika.XML;
import static org.alfresco.transform.tika.transformers.Tika.ZIP; import static org.alfresco.transform.tika.transformers.Tika.ZIP;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.ByteArrayInputStream;
import static org.mockito.ArgumentMatchers.any; import java.io.File;
import static org.mockito.Mockito.when; import java.io.IOException;
import static org.springframework.http.HttpHeaders.ACCEPT; import java.util.UUID;
import static org.springframework.http.HttpHeaders.CONTENT_DISPOSITION;
import static org.springframework.http.HttpHeaders.CONTENT_TYPE; import com.google.common.collect.ImmutableSet;
import static org.springframework.http.HttpStatus.CREATED; import org.apache.poi.ooxml.POIXMLProperties;
import static org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR; import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import static org.springframework.http.HttpStatus.OK; import org.junit.jupiter.api.BeforeEach;
import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; import org.junit.jupiter.api.Test;
import static org.springframework.http.MediaType.APPLICATION_PDF_VALUE; import org.mockito.Mock;
import static org.springframework.http.MediaType.TEXT_PLAIN_VALUE; import org.springframework.core.io.FileSystemResource;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; import org.springframework.core.io.Resource;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import org.springframework.http.HttpHeaders;
import org.springframework.http.ResponseEntity;
import org.springframework.mock.web.MockMultipartFile;
import org.springframework.test.web.servlet.MvcResult;
import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder;
import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
import org.alfresco.transform.base.AbstractBaseTest;
import org.alfresco.transform.base.executors.RuntimeExec;
import org.alfresco.transform.base.model.FileRefEntity;
import org.alfresco.transform.base.model.FileRefResponse;
import org.alfresco.transform.client.model.TransformReply;
import org.alfresco.transform.client.model.TransformRequest;
/** /**
* Test Tika. * Test Tika.
@@ -113,9 +115,9 @@ public class TikaTest extends AbstractBaseTest
private static final String EXPECTED_XHTML_CONTENT_CONTAINS = "<p>The quick brown fox jumps over the lazy dog</p>"; private static final String EXPECTED_XHTML_CONTENT_CONTAINS = "<p>The quick brown fox jumps over the lazy dog</p>";
private static final String EXPECTED_TEXT_CONTENT_CONTAINS = "The quick brown fox jumps over the lazy dog"; private static final String EXPECTED_TEXT_CONTENT_CONTAINS = "The quick brown fox jumps over the lazy dog";
private static final String EXPECTED_MSG_CONTENT_CONTAINS = "Recipients\n" + private static final String EXPECTED_MSG_CONTENT_CONTAINS = "Recipients\n" +
"\tmark.rogers@alfresco.com; speedy@quick.com; mrquick@nowhere.com\n" + "\tmark.rogers@alfresco.com; speedy@quick.com; mrquick@nowhere.com\n" +
"\n" + "\n" +
"The quick brown fox jumps over the lazy dogs"; "The quick brown fox jumps over the lazy dogs";
private static final String EXPECTED_CSV_CONTENT_CONTAINS = "\"The\",\"quick\",\"brown\",\"fox\""; private static final String EXPECTED_CSV_CONTENT_CONTAINS = "\"The\",\"quick\",\"brown\",\"fox\"";
@Mock @Mock
@@ -139,8 +141,8 @@ public class TikaTest extends AbstractBaseTest
@Override @Override
protected void mockTransformCommand(String sourceExtension, protected void mockTransformCommand(String sourceExtension,
String targetExtension, String sourceMimetype, String targetExtension, String sourceMimetype,
boolean readTargetFileBytes) throws IOException boolean readTargetFileBytes) throws IOException
{ {
// Tika transform is not mocked. It is run for real. // Tika transform is not mocked. It is run for real.
@@ -160,8 +162,8 @@ public class TikaTest extends AbstractBaseTest
} }
private void transform(String transform, String sourceExtension, String targetExtension, private void transform(String transform, String sourceExtension, String targetExtension,
String sourceMimetype, String targetMimetype, String sourceMimetype, String targetMimetype,
Boolean includeContents, String expectedContentContains) throws Exception Boolean includeContents, String expectedContentContains) throws Exception
{ {
// We don't use targetFileBytes as some of the transforms contain different date text based on the os being used. // We don't use targetFileBytes as some of the transforms contain different date text based on the os being used.
mockTransformCommand(sourceExtension, targetExtension, sourceMimetype, false); mockTransformCommand(sourceExtension, targetExtension, sourceMimetype, false);
@@ -169,18 +171,18 @@ public class TikaTest extends AbstractBaseTest
System.out.println("Test " + transform + " " + sourceExtension + " to " + targetExtension); System.out.println("Test " + transform + " " + sourceExtension + " to " + targetExtension);
MockHttpServletRequestBuilder requestBuilder = includeContents == null MockHttpServletRequestBuilder requestBuilder = includeContents == null
? mockMvcRequest(ENDPOINT_TRANSFORM, sourceFile, ? mockMvcRequest(ENDPOINT_TRANSFORM, sourceFile,
"targetExtension", this.targetExtension) "targetExtension", this.targetExtension)
: mockMvcRequest(ENDPOINT_TRANSFORM, sourceFile, : mockMvcRequest(ENDPOINT_TRANSFORM, sourceFile,
"targetExtension", this.targetExtension, INCLUDE_CONTENTS, includeContents.toString()); "targetExtension", this.targetExtension, INCLUDE_CONTENTS, includeContents.toString());
MvcResult result = mockMvc.perform(requestBuilder) MvcResult result = mockMvc.perform(requestBuilder)
.andExpect(status().is(OK.value())) .andExpect(status().is(OK.value()))
.andExpect(header().string("Content-Disposition", .andExpect(header().string("Content-Disposition",
"attachment; filename*=UTF-8''transform." + this.targetExtension)) "attachment; filename*=UTF-8''transform." + this.targetExtension))
.andReturn(); .andReturn();
String content = result.getResponse().getContentAsString(); String content = result.getResponse().getContentAsString();
assertTrue(content.contains(expectedContentContains), assertTrue(content.contains(expectedContentContains),
"The content did not include \"" + expectedContentContains); "The content did not include \"" + expectedContentContains);
} }
@Override @Override
@@ -188,9 +190,9 @@ public class TikaTest extends AbstractBaseTest
protected MockHttpServletRequestBuilder mockMvcRequest(String url, MockMultipartFile sourceFile, String... params) protected MockHttpServletRequestBuilder mockMvcRequest(String url, MockMultipartFile sourceFile, String... params)
{ {
return super.mockMvcRequest(url, sourceFile, params) return super.mockMvcRequest(url, sourceFile, params)
.param("targetEncoding", targetEncoding) .param("targetEncoding", targetEncoding)
.param("targetMimetype", targetMimetype) .param("targetMimetype", targetMimetype)
.param("sourceMimetype", sourceMimetype); .param("sourceMimetype", sourceMimetype);
} }
@Test @Test
@@ -199,8 +201,8 @@ public class TikaTest extends AbstractBaseTest
mockTransformCommand(PDF, TXT, MIMETYPE_PDF, true); mockTransformCommand(PDF, TXT, MIMETYPE_PDF, true);
targetEncoding = "rubbish"; targetEncoding = "rubbish";
mockMvc.perform( mockMvc.perform(
mockMvcRequest(ENDPOINT_TRANSFORM, sourceFile, "targetExtension", targetExtension)) mockMvcRequest(ENDPOINT_TRANSFORM, sourceFile, "targetExtension", targetExtension))
.andExpect(status().is(INTERNAL_SERVER_ERROR.value())); .andExpect(status().is(INTERNAL_SERVER_ERROR.value()));
} }
// --- Archive --- // --- Archive ---
@@ -209,55 +211,55 @@ public class TikaTest extends AbstractBaseTest
public void zipToTextArchiveTest() throws Exception public void zipToTextArchiveTest() throws Exception
{ {
transform(ARCHIVE, ZIP, TXT, MIMETYPE_ZIP, MIMETYPE_TEXT_PLAIN, false, transform(ARCHIVE, ZIP, TXT, MIMETYPE_ZIP, MIMETYPE_TEXT_PLAIN, false,
"quick.html\n" + "quick.html\n" +
"\n" + "\n" +
"\n" + "\n" +
"quick.pdf\n" + "quick.pdf\n" +
"\n" + "\n" +
"\n"); "\n");
} }
@Test @Test
public void zipToTextIncludeArchiveTest() throws Exception public void zipToTextIncludeArchiveTest() throws Exception
{ {
transform(ARCHIVE, ZIP, TXT, MIMETYPE_ZIP, MIMETYPE_TEXT_PLAIN, true, transform(ARCHIVE, ZIP, TXT, MIMETYPE_ZIP, MIMETYPE_TEXT_PLAIN, true,
"quick.html\n" + "quick.html\n" +
"\n" + "\n" +
"\n" + "\n" +
"The quick brown fox jumps over the lazy dog\n" + "The quick brown fox jumps over the lazy dog\n" +
"\n" + "\n" +
"\n" + "\n" +
"\n" + "\n" +
"quick.pdf\n" + "quick.pdf\n" +
"\n" + "\n" +
"\n" + "\n" +
"The quick brown fox jumps over the lazy dog" + "The quick brown fox jumps over the lazy dog" +
"\n" + "\n" +
"\n"); "\n");
} }
@Test @Test
public void zipToTextExcludeArchiveTest() throws Exception public void zipToTextExcludeArchiveTest() throws Exception
{ {
transform(ARCHIVE, ZIP, TXT, MIMETYPE_ZIP, MIMETYPE_TEXT_PLAIN, transform(ARCHIVE, ZIP, TXT, MIMETYPE_ZIP, MIMETYPE_TEXT_PLAIN,
false, "\n" + false, "\n" +
"folder/subfolder/quick.jpg\n" + "folder/subfolder/quick.jpg\n" +
"\n" + "\n" +
"\n" + "\n" +
"quick.doc\n" + "quick.doc\n" +
"\n" + "\n" +
"\n" + "\n" +
"quick.html\n" + "quick.html\n" +
"\n" + "\n" +
"\n" + "\n" +
"quick.pdf\n" + "quick.pdf\n" +
"\n" + "\n" +
"\n" + "\n" +
"quick.txt\n" + "quick.txt\n" +
"\n" + "\n" +
"\n" + "\n" +
"quick.xml\n" + "quick.xml\n" +
"\n"); "\n");
} }
// --- OutlookMsg --- // --- OutlookMsg ---
@@ -266,7 +268,7 @@ public class TikaTest extends AbstractBaseTest
public void msgToTxtOutlookMsgTest() throws Exception public void msgToTxtOutlookMsgTest() throws Exception
{ {
transform(OUTLOOK_MSG, MSG, TXT, MIMETYPE_OUTLOOK_MSG, MIMETYPE_TEXT_PLAIN, null, transform(OUTLOOK_MSG, MSG, TXT, MIMETYPE_OUTLOOK_MSG, MIMETYPE_TEXT_PLAIN, null,
EXPECTED_MSG_CONTENT_CONTAINS); EXPECTED_MSG_CONTENT_CONTAINS);
} }
// --- PdfBox --- // --- PdfBox ---
@@ -275,35 +277,35 @@ public class TikaTest extends AbstractBaseTest
public void pdfToTxtPdfBoxTest() throws Exception public void pdfToTxtPdfBoxTest() throws Exception
{ {
transform(PDF_BOX, PDF, TXT, MIMETYPE_PDF, MIMETYPE_TEXT_PLAIN, null, transform(PDF_BOX, PDF, TXT, MIMETYPE_PDF, MIMETYPE_TEXT_PLAIN, null,
EXPECTED_TEXT_CONTENT_CONTAINS); EXPECTED_TEXT_CONTENT_CONTAINS);
} }
@Test @Test
public void pdfToCsvPdfBoxTest() throws Exception public void pdfToCsvPdfBoxTest() throws Exception
{ {
transform(PDF_BOX, PDF, CSV, MIMETYPE_PDF, MIMETYPE_TEXT_CSV, null, transform(PDF_BOX, PDF, CSV, MIMETYPE_PDF, MIMETYPE_TEXT_CSV, null,
EXPECTED_TEXT_CONTENT_CONTAINS); // Yes it is just text EXPECTED_TEXT_CONTENT_CONTAINS); // Yes it is just text
} }
@Test @Test
public void pdfToXmlPdfBoxTest() throws Exception public void pdfToXmlPdfBoxTest() throws Exception
{ {
transform(PDF_BOX, PDF, XML, MIMETYPE_PDF, MIMETYPE_XML, null, transform(PDF_BOX, PDF, XML, MIMETYPE_PDF, MIMETYPE_XML, null,
EXPECTED_XHTML_CONTENT_CONTAINS); // Yes it is just XHTML EXPECTED_XHTML_CONTENT_CONTAINS); // Yes it is just XHTML
} }
@Test @Test
public void pdfToXhtmlPdfBoxTest() throws Exception public void pdfToXhtmlPdfBoxTest() throws Exception
{ {
transform(PDF_BOX, PDF, XHTML, MIMETYPE_PDF, MIMETYPE_XHTML, null, transform(PDF_BOX, PDF, XHTML, MIMETYPE_PDF, MIMETYPE_XHTML, null,
EXPECTED_XHTML_CONTENT_CONTAINS); EXPECTED_XHTML_CONTENT_CONTAINS);
} }
@Test @Test
public void pdfToHtmlPdfBoxTest() throws Exception public void pdfToHtmlPdfBoxTest() throws Exception
{ {
transform(PDF_BOX, PDF, HTML, MIMETYPE_PDF, MIMETYPE_HTML, null, transform(PDF_BOX, PDF, HTML, MIMETYPE_PDF, MIMETYPE_HTML, null,
EXPECTED_XHTML_CONTENT_CONTAINS); // Yes it is just XHTML EXPECTED_XHTML_CONTENT_CONTAINS); // Yes it is just XHTML
} }
// --- Office --- // --- Office ---
@@ -312,14 +314,14 @@ public class TikaTest extends AbstractBaseTest
public void msgToTxtOfficeTest() throws Exception public void msgToTxtOfficeTest() throws Exception
{ {
transform(OFFICE, MSG, TXT, MIMETYPE_OUTLOOK_MSG, MIMETYPE_TEXT_PLAIN, null, transform(OFFICE, MSG, TXT, MIMETYPE_OUTLOOK_MSG, MIMETYPE_TEXT_PLAIN, null,
EXPECTED_MSG_CONTENT_CONTAINS); EXPECTED_MSG_CONTENT_CONTAINS);
} }
@Test @Test
public void docToTxtOfficeTest() throws Exception public void docToTxtOfficeTest() throws Exception
{ {
transform(OFFICE, DOC, TXT, MIMETYPE_WORD, MIMETYPE_TEXT_PLAIN, null, transform(OFFICE, DOC, TXT, MIMETYPE_WORD, MIMETYPE_TEXT_PLAIN, null,
EXPECTED_TEXT_CONTENT_CONTAINS); EXPECTED_TEXT_CONTENT_CONTAINS);
} }
// --- Poi --- // --- Poi ---
@@ -328,7 +330,7 @@ public class TikaTest extends AbstractBaseTest
public void xslxToCsvPoiTest() throws Exception public void xslxToCsvPoiTest() throws Exception
{ {
transform(POI, XLSX, CSV, MIMETYPE_OPENXML_SPREADSHEET, MIMETYPE_TEXT_CSV, null, transform(POI, XLSX, CSV, MIMETYPE_OPENXML_SPREADSHEET, MIMETYPE_TEXT_CSV, null,
EXPECTED_CSV_CONTENT_CONTAINS); EXPECTED_CSV_CONTENT_CONTAINS);
} }
// --- OOXML --- // --- OOXML ---
@@ -337,14 +339,14 @@ public class TikaTest extends AbstractBaseTest
public void docxToTxtOoXmlTest() throws Exception public void docxToTxtOoXmlTest() throws Exception
{ {
transform(OOXML, DOCX, TXT, MIMETYPE_OPENXML_WORDPROCESSING, MIMETYPE_TEXT_PLAIN, null, transform(OOXML, DOCX, TXT, MIMETYPE_OPENXML_WORDPROCESSING, MIMETYPE_TEXT_PLAIN, null,
EXPECTED_TEXT_CONTENT_CONTAINS); EXPECTED_TEXT_CONTENT_CONTAINS);
} }
@Test @Test
public void pptxToTxtOoXmlTest() throws Exception public void pptxToTxtOoXmlTest() throws Exception
{ {
transform(OOXML, PPTX, TXT, MIMETYPE_OPENXML_PRESENTATION, MIMETYPE_TEXT_PLAIN, null, transform(OOXML, PPTX, TXT, MIMETYPE_OPENXML_PRESENTATION, MIMETYPE_TEXT_PLAIN, null,
EXPECTED_TEXT_CONTENT_CONTAINS); EXPECTED_TEXT_CONTENT_CONTAINS);
} }
// --- TikaAuto --- // --- TikaAuto ---
@@ -353,14 +355,14 @@ public class TikaTest extends AbstractBaseTest
public void ppxtToTxtTikaAutoTest() throws Exception public void ppxtToTxtTikaAutoTest() throws Exception
{ {
transform(TIKA_AUTO, PPTX, TXT, MIMETYPE_OPENXML_PRESENTATION, MIMETYPE_TEXT_PLAIN, null, transform(TIKA_AUTO, PPTX, TXT, MIMETYPE_OPENXML_PRESENTATION, MIMETYPE_TEXT_PLAIN, null,
EXPECTED_TEXT_CONTENT_CONTAINS); EXPECTED_TEXT_CONTENT_CONTAINS);
} }
@Test @Test
public void doctToTxtTikaAutoTest() throws Exception public void doctToTxtTikaAutoTest() throws Exception
{ {
transform(TIKA_AUTO, DOCX, TXT, MIMETYPE_OPENXML_WORDPROCESSING, MIMETYPE_TEXT_PLAIN, null, transform(TIKA_AUTO, DOCX, TXT, MIMETYPE_OPENXML_WORDPROCESSING, MIMETYPE_TEXT_PLAIN, null,
EXPECTED_TEXT_CONTENT_CONTAINS); EXPECTED_TEXT_CONTENT_CONTAINS);
} }
// --- TextMining --- // --- TextMining ---
@@ -369,7 +371,7 @@ public class TikaTest extends AbstractBaseTest
public void docToTxtTextMiningTest() throws Exception public void docToTxtTextMiningTest() throws Exception
{ {
transform(TEXT_MINING, DOC, TXT, MIMETYPE_WORD, MIMETYPE_TEXT_PLAIN, null, transform(TEXT_MINING, DOC, TXT, MIMETYPE_WORD, MIMETYPE_TEXT_PLAIN, null,
EXPECTED_TEXT_CONTENT_CONTAINS); EXPECTED_TEXT_CONTENT_CONTAINS);
} }
@Test @Test
@@ -377,24 +379,22 @@ public class TikaTest extends AbstractBaseTest
{ {
mockTransformCommand(XLSX, XLSX, MIMETYPE_OPENXML_SPREADSHEET, false); mockTransformCommand(XLSX, XLSX, MIMETYPE_OPENXML_SPREADSHEET, false);
String metadata = String metadata = "{\"{http://www.alfresco.org/model/content/1.0}author\":\"author1\"," +
"{\"{http://www.alfresco.org/model/content/1.0}author\":\"author1\"," + "\"{http://www.alfresco.org/model/content/1.0}title\":\"title1\"," +
"\"{http://www.alfresco.org/model/content/1.0}title\":\"title1\"," + "\"{http://www.alfresco.org/model/content/1.0}description\":[\"desc1\",\"desc2\"]," +
"\"{http://www.alfresco.org/model/content/1.0}description\":[\"desc1\",\"desc2\"]," + "\"{http://www.alfresco.org/model/content/1.0}created\":\"created1\"}";
"\"{http://www.alfresco.org/model/content/1.0}created\":\"created1\"}";
MockHttpServletRequestBuilder requestBuilder = MockHttpServletRequestBuilder requestBuilder = super.mockMvcRequest(ENDPOINT_TRANSFORM, sourceFile,
super.mockMvcRequest(ENDPOINT_TRANSFORM, sourceFile, "targetExtension", XLSX,
"targetExtension", XLSX, "metadata", metadata,
"metadata", metadata, "targetMimetype", MIMETYPE_METADATA_EMBED,
"targetMimetype", MIMETYPE_METADATA_EMBED, "sourceMimetype", MIMETYPE_OPENXML_SPREADSHEET);
"sourceMimetype", MIMETYPE_OPENXML_SPREADSHEET);
MvcResult result = mockMvc.perform(requestBuilder) MvcResult result = mockMvc.perform(requestBuilder)
.andExpect(status().is(OK.value())) .andExpect(status().is(OK.value()))
.andExpect(header().string("Content-Disposition", .andExpect(header().string("Content-Disposition",
"attachment; filename*=UTF-8''transform." + targetExtension)). "attachment; filename*=UTF-8''transform." + targetExtension))
andReturn(); .andReturn();
byte[] bytes = result.getResponse().getContentAsByteArray(); byte[] bytes = result.getResponse().getContentAsByteArray();
ByteArrayInputStream inputStream = new ByteArrayInputStream(bytes); ByteArrayInputStream inputStream = new ByteArrayInputStream(bytes);
@@ -414,11 +414,11 @@ public class TikaTest extends AbstractBaseTest
{ {
mockTransformCommand(PDF, TXT, MIMETYPE_PDF, true); mockTransformCommand(PDF, TXT, MIMETYPE_PDF, true);
mockMvc.perform( mockMvc.perform(
mockMvcRequest(ENDPOINT_TRANSFORM, sourceFile, "targetExtension", targetExtension).param( mockMvcRequest(ENDPOINT_TRANSFORM, sourceFile, "targetExtension", targetExtension).param(
NOT_EXTRACT_BOOKMARKS_TEXT, "true")) NOT_EXTRACT_BOOKMARKS_TEXT, "true"))
.andExpect(status().is(OK.value())) .andExpect(status().is(OK.value()))
.andExpect(header().string("Content-Disposition", .andExpect(header().string("Content-Disposition",
"attachment; filename*=UTF-8''transform." + targetExtension)); "attachment; filename*=UTF-8''transform." + targetExtension));
} }
@Override @Override
@@ -445,11 +445,11 @@ public class TikaTest extends AbstractBaseTest
HttpHeaders headers = new HttpHeaders(); HttpHeaders headers = new HttpHeaders();
headers.set(CONTENT_DISPOSITION, "attachment; filename=quick." + sourceExtension); headers.set(CONTENT_DISPOSITION, "attachment; filename=quick." + sourceExtension);
ResponseEntity<Resource> response = new ResponseEntity<>(new FileSystemResource( ResponseEntity<Resource> response = new ResponseEntity<>(new FileSystemResource(
sourceFile), headers, OK); sourceFile), headers, OK);
when(sharedFileStoreClient.retrieveFile(sourceFileRef)).thenReturn(response); when(sharedFileStoreClient.retrieveFile(sourceFileRef)).thenReturn(response);
when(sharedFileStoreClient.saveFile(any())) when(sharedFileStoreClient.saveFile(any()))
.thenReturn(new FileRefResponse(new FileRefEntity(targetFileRef))); .thenReturn(new FileRefResponse(new FileRefEntity(targetFileRef)));
when(mockExecutionResult.getExitValue()).thenReturn(0); when(mockExecutionResult.getExitValue()).thenReturn(0);
// Update the Transformation Request with any specific params before sending it // Update the Transformation Request with any specific params before sending it
@@ -458,16 +458,16 @@ public class TikaTest extends AbstractBaseTest
// Serialize and call the transformer // Serialize and call the transformer
String tr = objectMapper.writeValueAsString(transformRequest); String tr = objectMapper.writeValueAsString(transformRequest);
String transformationReplyAsString = mockMvc String transformationReplyAsString = mockMvc
.perform(MockMvcRequestBuilders .perform(MockMvcRequestBuilders
.post(ENDPOINT_TRANSFORM) .post(ENDPOINT_TRANSFORM)
.header(ACCEPT, APPLICATION_JSON_VALUE) .header(ACCEPT, APPLICATION_JSON_VALUE)
.header(CONTENT_TYPE, APPLICATION_JSON_VALUE) .header(CONTENT_TYPE, APPLICATION_JSON_VALUE)
.content(tr)) .content(tr))
.andExpect(status().is(CREATED.value())) .andExpect(status().is(CREATED.value()))
.andReturn().getResponse().getContentAsString(); .andReturn().getResponse().getContentAsString();
TransformReply transformReply = objectMapper.readValue(transformationReplyAsString, TransformReply transformReply = objectMapper.readValue(transformationReplyAsString,
TransformReply.class); TransformReply.class);
// Assert the reply // Assert the reply
assertEquals(transformRequest.getRequestId(), transformReply.getRequestId()); assertEquals(transformRequest.getRequestId(), transformReply.getRequestId());
@@ -492,6 +492,6 @@ public class TikaTest extends AbstractBaseTest
"extractMapping", "extractMapping",
"notExtractBookmarksText", "notExtractBookmarksText",
"metadata"), "metadata"),
getOptionNames(controller.transformConfig(0).getBody().getTransformOptions())); getOptionNames(controller.transformConfig(0).getBody().getTransformOptions()));
} }
} }

189
model/src/main/java/org/alfresco/transform/common/RequestParamMap.java
View File

@@ -1,93 +1,96 @@
/* /*
* #%L * #%L
* Alfresco Transform Model * Alfresco Transform Model
* %% * %%
* Copyright (C) 2005 - 2022 Alfresco Software Limited * Copyright (C) 2005 - 2025 Alfresco Software Limited
* %% * %%
* This program is free software: you can redistribute it and/or modify * This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as * it under the terms of the GNU Lesser General Public License as
* published by the Free Software Foundation, either version 3 of the * published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version. * License, or (at your option) any later version.
* *
* This program is distributed in the hope that it will be useful, * This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of * but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Lesser Public License for more details. * GNU General Lesser Public License for more details.
* *
* You should have received a copy of the GNU General Lesser Public * You should have received a copy of the GNU General Lesser Public
* License along with this program. If not, see * License along with this program. If not, see
* <http://www.gnu.org/licenses/lgpl-3.0.html>. * <http://www.gnu.org/licenses/lgpl-3.0.html>.
* #L% * #L%
*/ */
package org.alfresco.transform.common; package org.alfresco.transform.common;
import org.alfresco.transform.config.CoreVersionDecorator; import org.alfresco.transform.config.CoreVersionDecorator;
/** /**
* Request parameters and transform options used in the core transformers. * Request parameters and transform options used in the core transformers.
*/ */
public interface RequestParamMap public interface RequestParamMap
{ {
// html parameter names // html parameter names
String FILE = "file"; String FILE = "file";
String SOURCE_EXTENSION = "sourceExtension"; String SOURCE_EXTENSION = "sourceExtension";
String TARGET_EXTENSION = "targetExtension"; String TARGET_EXTENSION = "targetExtension";
String SOURCE_MIMETYPE = "sourceMimetype"; String SOURCE_MIMETYPE = "sourceMimetype";
String TARGET_MIMETYPE = "targetMimetype"; String TARGET_MIMETYPE = "targetMimetype";
// Transform options used in the core transformers. // Transform options used in the core transformers.
String SOURCE_ENCODING = "sourceEncoding"; String SOURCE_ENCODING = "sourceEncoding";
String TARGET_ENCODING = "targetEncoding"; String TARGET_ENCODING = "targetEncoding";
String PAGE_REQUEST_PARAM = "page"; String PAGE_REQUEST_PARAM = "page";
String WIDTH_REQUEST_PARAM = "width"; String WIDTH_REQUEST_PARAM = "width";
String HEIGHT_REQUEST_PARAM = "height"; String HEIGHT_REQUEST_PARAM = "height";
String ALLOW_PDF_ENLARGEMENT = "allowPdfEnlargement"; String ALLOW_PDF_ENLARGEMENT = "allowPdfEnlargement";
String MAINTAIN_PDF_ASPECT_RATIO = "maintainPdfAspectRatio"; String MAINTAIN_PDF_ASPECT_RATIO = "maintainPdfAspectRatio";
String START_PAGE = "startPage"; String START_PAGE = "startPage";
String END_PAGE = "endPage"; String END_PAGE = "endPage";
String ALPHA_REMOVE = "alphaRemove"; String ALPHA_REMOVE = "alphaRemove";
String AUTO_ORIENT = "autoOrient"; String AUTO_ORIENT = "autoOrient";
String CROP_GRAVITY = "cropGravity"; String CROP_GRAVITY = "cropGravity";
String CROP_WIDTH = "cropWidth"; String CROP_WIDTH = "cropWidth";
String CROP_HEIGHT = "cropHeight"; String CROP_HEIGHT = "cropHeight";
String CROP_PERCENTAGE = "cropPercentage"; String CROP_PERCENTAGE = "cropPercentage";
String CROP_X_OFFSET = "cropXOffset"; String CROP_X_OFFSET = "cropXOffset";
String CROP_Y_OFFSET = "cropYOffset"; String CROP_Y_OFFSET = "cropYOffset";
String THUMBNAIL = "thumbnail"; String THUMBNAIL = "thumbnail";
String RESIZE_WIDTH = "resizeWidth"; String RESIZE_WIDTH = "resizeWidth";
String RESIZE_HEIGHT = "resizeHeight"; String RESIZE_HEIGHT = "resizeHeight";
String RESIZE_PERCENTAGE = "resizePercentage"; String RESIZE_PERCENTAGE = "resizePercentage";
String ALLOW_ENLARGEMENT = "allowEnlargement"; String ALLOW_ENLARGEMENT = "allowEnlargement";
String MAINTAIN_ASPECT_RATIO = "maintainAspectRatio"; String MAINTAIN_ASPECT_RATIO = "maintainAspectRatio";
String COMMAND_OPTIONS = "commandOptions"; String COMMAND_OPTIONS = "commandOptions";
String TIMEOUT = "timeout"; String TIMEOUT = "timeout";
String INCLUDE_CONTENTS = "includeContents"; String INCLUDE_CONTENTS = "includeContents";
String NOT_EXTRACT_BOOKMARKS_TEXT = "notExtractBookmarksText"; String NOT_EXTRACT_BOOKMARKS_TEXT = "notExtractBookmarksText";
String PAGE_LIMIT = "pageLimit"; String PAGE_LIMIT = "pageLimit";
String PDF_FORMAT = "pdfFormat"; String PDF_FORMAT = "pdfFormat";
String PDF_ORIENTATION = "pdfOrientation"; String PDF_ORIENTATION = "pdfOrientation";
String PDF_FONT = "pdfFont"; String PDF_FONT = "pdfFont";
String PDF_FONT_SIZE = "pdfFontSize"; String PDF_FONT_SIZE = "pdfFontSize";
// Parameters interpreted by the TransformController // Html parameter names for the transform config
String DIRECT_ACCESS_URL = "directAccessUrl"; String HTML_COLLAPSE = "collapseHtml";
// An optional parameter (defaults to 1) to be included in the request to the t-engine {@code /transform/config} // Parameters interpreted by the TransformController
// endpoint to specify what version (of the schema) to return. Provides the flexibility to introduce changes String DIRECT_ACCESS_URL = "directAccessUrl";
// without getting deserialization issues when we have components at different versions.
String CONFIG_VERSION = "configVersion"; // An optional parameter (defaults to 1) to be included in the request to the t-engine {@code /transform/config}
String CONFIG_VERSION_DEFAULT = "1"; // endpoint to specify what version (of the schema) to return. Provides the flexibility to introduce changes
int CONFIG_VERSION_LATEST = CoreVersionDecorator.CONFIG_VERSION_INCLUDES_CORE_VERSION; // without getting deserialization issues when we have components at different versions.
String CONFIG_VERSION = "configVersion";
// Endpoints String CONFIG_VERSION_DEFAULT = "1";
String ENDPOINT_TRANSFORM = "/transform"; int CONFIG_VERSION_LATEST = CoreVersionDecorator.CONFIG_VERSION_INCLUDES_CORE_VERSION;
String ENDPOINT_TEST = "/test";
String ENDPOINT_TRANSFORM_CONFIG = "/transform/config"; // Endpoints
String ENDPOINT_TRANSFORM_CONFIG_LATEST = ENDPOINT_TRANSFORM_CONFIG + "?" + CONFIG_VERSION + "=" + CONFIG_VERSION_LATEST; String ENDPOINT_TRANSFORM = "/transform";
String ENDPOINT_VERSION = "/version"; String ENDPOINT_TEST = "/test";
String ENDPOINT_READY = "/ready"; String ENDPOINT_TRANSFORM_CONFIG = "/transform/config";
String ENDPOINT_LIVE = "/live"; String ENDPOINT_TRANSFORM_CONFIG_LATEST = ENDPOINT_TRANSFORM_CONFIG + "?" + CONFIG_VERSION + "=" + CONFIG_VERSION_LATEST;
String ENDPOINT_ERROR = "/error"; String ENDPOINT_VERSION = "/version";
String ENDPOINT_LOG = "/log"; String ENDPOINT_READY = "/ready";
String ENDPOINT_ROOT = "/"; String ENDPOINT_LIVE = "/live";
} String ENDPOINT_ERROR = "/error";
String ENDPOINT_LOG = "/log";
String ENDPOINT_ROOT = "/";
}