From b619f272074059f9281f49de7bc386933ab1859d Mon Sep 17 00:00:00 2001 From: alandavis Date: Thu, 30 Jun 2022 13:39:24 +0100 Subject: [PATCH] Save point: [skip ci] * Beginnings of new t-base (using TransformEngine and CustomeTransformer, no need for a controller of Application in t-engine modules) * Using org.alfresco.transform. package * Beginnings of new Tika t-engine --- .../alfresco-transform-core-aio-boot/pom.xml | 3 + .../aio}/AIOController.java | 11 +- .../aio}/AIOCustomConfig.java | 17 +- .../org/alfresco/transformer/Application.java | 85 -- .../aio}/AIOControllerHttpRequestTest.java | 7 +- .../aio}/AIOControllerImageMagickTest.java | 16 +- .../aio}/AIOControllerLibreOfficeTest.java | 16 +- .../aio}/AIOControllerMiscTest.java | 9 +- .../aio}/AIOControllerPdfRendererTest.java | 13 +- .../aio}/AIOControllerTest.java | 5 +- .../aio}/AIOControllerTikaTest.java | 7 +- .../aio}/AIOCustomConfigTest.java | 6 +- .../aio}/AIOImageMagickIT.java | 6 +- .../aio}/AIOLibreOfficeTransformationIT.java | 6 +- .../aio}/AIOMiscMetadataExtractsIT.java | 6 +- .../aio}/AIOMiscTransformsIT.java | 6 +- .../aio}/AIOPdfRendererIT.java | 6 +- .../aio}/AIOQueueTransformServiceIT.java | 3 +- .../aio}/AIOTikaMetadataExtractsIT.java | 6 +- .../aio}/AIOTikaTransformationIT.java | 6 +- .../aio}/AIOTransformRegistry.java | 4 +- .../aio}/AIOTransformRegistryTest.java | 13 +- .../pom.xml | 3 + .../imagemagick}/Application.java | 6 +- .../imagemagick}/ImageMagickController.java | 5 +- .../ImageMagickControllerTest.java | 6 +- .../ImageMagickHttpRequestTest.java | 5 +- .../ImageMagickQueueTransformServiceIT.java | 3 +- .../ImageMagickTransformationIT.java | 3 +- .../ImageMagickOptionsBuilder.java | 2 +- .../ImageMagickCommandExecutor.java | 6 +- .../pom.xml | 3 + .../office}/Application.java | 6 +- .../office}/LibreOfficeController.java | 5 +- .../office}/LibreOfficeControllerTest.java | 6 +- .../office}/LibreOfficeHttpRequestTest.java | 5 +- .../LibreOfficeQueueTransformServiceIT.java | 5 +- .../office}/LibreOfficeTransformationIT.java | 6 +- .../office/transformers}/JodConverter.java | 4 +- .../JodConverterSharedInstance.java | 4 +- .../LibreOfficeExtractMetadataTask.java | 4 +- .../LibreOfficeJavaExecutor.java | 3 +- .../alfresco-transform-misc-boot/pom.xml | 3 + .../misc}/Application.java | 6 +- .../misc}/MiscController.java | 6 +- .../misc}/MiscControllerTest.java | 4 +- .../misc}/MiscMetadataExtractsIT.java | 4 +- .../misc}/MiscQueueTransformServiceIT.java | 3 +- .../misc}/MiscTransformerHttpRequestTest.java | 5 +- .../misc}/MiscTransformsIT.java | 4 +- .../HtmlMetadataExtractor.java | 4 +- .../RFC822MetadataExtractor.java | 4 +- .../AppleIWorksContentTransformer.java | 2 +- .../misc}/transformers/EMLTransformer.java | 2 +- .../HtmlParserContentTransformer.java | 2 +- .../OOXMLThumbnailContentTransformer.java | 4 +- .../transformers/SelectableTransformer.java | 2 +- .../transformers/SelectingTransformer.java | 4 +- .../StringExtractingContentTransformer.java | 2 +- .../TextToPdfContentTransformer.java | 2 +- .../HtmlParserContentTransformerTest.java | 2 +- .../TextToPdfContentTransformerTest.java | 2 +- .../common/TransformConfigResourceReader.java | 63 ++ .../pom.xml | 3 + .../AlfrescoPdfRendererController.java | 5 +- .../pdfRenderer}/Application.java | 6 +- .../AlfrescoPdfRendererControllerTest.java | 6 +- .../AlfrescoPdfRendererHttpRequestTest.java | 5 +- ...scoPdfRendererQueueTransformServiceIT.java | 5 +- .../AlfrescoPdfRendererTransformationIT.java | 5 +- .../PdfRendererOptionsBuilder.java | 4 +- .../PdfRendererCommandExecutor.java | 6 +- .../alfresco-transform-tika-boot/pom.xml | 7 +- .../alfresco/transformer/TikaController.java | 111 -- .../tika}/TikaControllerTest.java | 78 +- .../tika}/TikaHttpRequestTest.java | 8 +- .../tika}/TikaMetadataExtractsIT.java | 6 +- .../tika}/TikaQueueTransformServiceIT.java | 3 +- .../tika}/TikaTransformationIT.java | 4 +- .../alfresco-transform-tika/pom.xml | 2 +- .../transform/tika/TikaTransformEngine.java | 78 ++ .../AbstractTikaMetadataExtractor.java | 48 +- .../DWGMetadataExtractor.java | 10 +- .../IPTCMetadataExtractor.java | 12 +- .../MP3MetadataExtractor.java | 8 +- .../MailMetadataExtractor.java | 10 +- .../OfficeMetadataExtractor.java | 10 +- .../OpenDocumentMetadataExtractor.java | 9 +- .../PdfBoxMetadataExtractor.java | 12 +- .../PoiMetadataExtractor.java | 10 +- .../TikaAudioMetadataExtractor.java | 11 +- .../TikaAutoMetadataExtractor.java | 10 +- .../tika/parsers/ExifToolParser.java | 2 +- .../tika/parsers}/TikaOfficeDetectParser.java | 4 +- .../tika/transformers/ArchiveTransformer.java | 42 + .../transformers/GenericTikaTransformer.java | 146 +++ .../tika/transformers/OOXMLTransformer.java | 40 + .../tika/transformers/OfficeTransformer.java | 40 + .../transformers/OutlookMsgTransformer.java | 40 + .../tika/transformers/PdfBoxTransformer.java | 47 + .../tika/transformers/PoiTransformer.java | 40 + .../transformers/TextMiningTransformer.java | 40 + .../transform/tika/transformers/Tika.java | 446 ++++++++ .../transformers/TikaAutoTransformer.java | 40 + .../alfresco/transformer/executors/Tika.java | 876 ---------------- .../executors/TikaJavaExecutor.java | 204 ---- .../IPTCMetadataExtractorTest.java | 10 +- .../tika/parsers/ExifToolParserTest.java | 4 +- .../GenericTikaTransformerTest.java} | 32 +- .../org/alfresco/transformer/util/Util.java | 3 - pom.xml | 8 + t-engine-base/README.md | 233 +++++ t-engine-base/pom.xml | 101 ++ .../alfresco/transform/base}/Application.java | 20 +- .../transform/base/CustomTransformer.java | 40 + .../transform/base/QueueTransformService.java | 208 ++++ .../transform/base/TransformController.java | 662 ++++++++++++ .../transform/base/TransformEngine.java | 56 + .../transform/base/TransformInterceptor.java | 64 ++ .../transform/base/TransformRegistryImpl.java | 106 ++ .../AlfrescoSharedFileStoreClient.java | 103 ++ .../base/config/WebApplicationConfig.java | 95 ++ .../executors/AbstractCommandExecutor.java | 91 ++ .../base/executors/CommandExecutor.java | 71 ++ .../executors/ExecParameterTokenizer.java | 360 +++++++ .../base/executors/JavaExecutor.java | 40 + .../transform/base/executors/RuntimeExec.java | 986 ++++++++++++++++++ .../transform/base/executors/Transformer.java | 130 +++ .../transform/base/fs/FileManager.java | 249 +++++ .../transform/base/logging/LogEntry.java | 285 +++++ .../base/logging/StandardMessages.java | 35 + .../base/messaging/MessagingConfig.java | 108 ++ .../base/messaging/MessagingInfo.java | 67 ++ .../messaging/TransformMessageConverter.java | 102 ++ .../base/messaging/TransformReplySender.java | 77 ++ .../AbstractMetadataExtractor.java | 599 +++++++++++ .../transform/base/model/FileRefEntity.java | 77 ++ .../transform/base/model/FileRefResponse.java | 54 + .../base/probes/ProbeTestTransform.java | 391 +++++++ .../transform/base/util/RequestParamMap.java | 41 + .../alfresco/transform/base/util/Util.java | 63 ++ .../src/main/resources/application.yaml | 61 ++ .../src/main/resources/engine_config.json | 1 + .../src/main/resources/templates/error.html | 22 + .../src/main/resources/templates/log.html | 43 + .../base/AbstractHttpRequestTest.java | 177 ++++ .../base/AbstractMetadataExtractsIT.java | 134 +++ .../base/AbstractQueueTransformServiceIT.java | 74 ++ .../AbstractTransformerControllerTest.java | 686 ++++++++++++ .../alfresco/transform/base/EngineClient.java | 69 ++ .../base/QueueTransformServiceTest.java | 240 +++++ .../alfresco/transform/base/SourceTarget.java | 76 ++ .../alfresco/transform/base/TestFileInfo.java | 85 ++ .../resources/engine_config_complete.json | 22 + .../resources/engine_config_incomplete.json | 10 + .../engine_config_no_transform_options.json | 10 + .../engine_config_with_duplicates.json | 26 + 157 files changed, 8740 insertions(+), 1543 deletions(-) rename alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/main/java/org/alfresco/{transformer => transform/aio}/AIOController.java (90%) rename alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/main/java/org/alfresco/{transformer => transform/aio}/AIOCustomConfig.java (86%) delete mode 100644 alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/main/java/org/alfresco/transformer/Application.java rename alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/{transformer => transform/aio}/AIOControllerHttpRequestTest.java (84%) rename alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/{transformer => transform/aio}/AIOControllerImageMagickTest.java (86%) rename alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/{transformer => transform/aio}/AIOControllerLibreOfficeTest.java (85%) rename alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/{transformer => transform/aio}/AIOControllerMiscTest.java (86%) rename alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/{transformer => transform/aio}/AIOControllerPdfRendererTest.java (87%) rename alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/{transformer => transform/aio}/AIOControllerTest.java (92%) rename alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/{transformer => transform/aio}/AIOControllerTikaTest.java (89%) rename alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/{transformer => transform/aio}/AIOCustomConfigTest.java (90%) rename alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/{transformer => transform/aio}/AIOImageMagickIT.java (84%) rename alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/{transformer => transform/aio}/AIOLibreOfficeTransformationIT.java (85%) rename alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/{transformer => transform/aio}/AIOMiscMetadataExtractsIT.java (89%) rename alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/{transformer => transform/aio}/AIOMiscTransformsIT.java (85%) rename alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/{transformer => transform/aio}/AIOPdfRendererIT.java (84%) rename alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/{transformer => transform/aio}/AIOQueueTransformServiceIT.java (92%) rename alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/{transformer => transform/aio}/AIOTikaMetadataExtractsIT.java (89%) rename alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/{transformer => transform/aio}/AIOTikaTransformationIT.java (85%) rename alfresco-transform-core-aio/alfresco-transform-core-aio/src/main/java/org/alfresco/{transformer => transform/aio}/AIOTransformRegistry.java (96%) rename alfresco-transform-core-aio/alfresco-transform-core-aio/src/test/java/org/alfresco/{transformer => transform/aio}/AIOTransformRegistryTest.java (96%) rename alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/main/java/org/alfresco/{transformer => transform/imagemagick}/Application.java (94%) rename alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/main/java/org/alfresco/{transformer => transform/imagemagick}/ImageMagickController.java (96%) rename alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/test/java/org/alfresco/{transformer => transform/imagemagick}/ImageMagickControllerTest.java (98%) rename alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/test/java/org/alfresco/{transformer => transform/imagemagick}/ImageMagickHttpRequestTest.java (90%) rename alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/test/java/org/alfresco/{transformer => transform/imagemagick}/ImageMagickQueueTransformServiceIT.java (95%) rename alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/test/java/org/alfresco/{transformer => transform/imagemagick}/ImageMagickTransformationIT.java (99%) rename alfresco-transform-imagemagick/alfresco-transform-imagemagick/src/main/java/org/alfresco/{transformer => transform/imagemagick}/ImageMagickOptionsBuilder.java (99%) rename alfresco-transform-imagemagick/alfresco-transform-imagemagick/src/main/java/org/alfresco/{transformer/executors => transform/imagemagick/transformers}/ImageMagickCommandExecutor.java (97%) rename alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/main/java/org/alfresco/{transformer => transform/office}/Application.java (95%) rename alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/main/java/org/alfresco/{transformer => transform/office}/LibreOfficeController.java (96%) rename alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/test/java/org/alfresco/{transformer => transform/office}/LibreOfficeControllerTest.java (98%) rename alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/test/java/org/alfresco/{transformer => transform/office}/LibreOfficeHttpRequestTest.java (90%) rename alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/test/java/org/alfresco/{transformer => transform/office}/LibreOfficeQueueTransformServiceIT.java (92%) rename alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/test/java/org/alfresco/{transformer => transform/office}/LibreOfficeTransformationIT.java (98%) rename alfresco-transform-libreoffice/alfresco-transform-libreoffice/src/main/java/org/alfresco/{transformer/executors => transform/office/transformers}/JodConverter.java (93%) rename alfresco-transform-libreoffice/alfresco-transform-libreoffice/src/main/java/org/alfresco/{transformer/executors => transform/office/transformers}/JodConverterSharedInstance.java (99%) rename alfresco-transform-libreoffice/alfresco-transform-libreoffice/src/main/java/org/alfresco/{transformer/executors => transform/office/transformers}/LibreOfficeExtractMetadataTask.java (98%) rename alfresco-transform-libreoffice/alfresco-transform-libreoffice/src/main/java/org/alfresco/{transformer/executors => transform/office/transformers}/LibreOfficeJavaExecutor.java (99%) rename alfresco-transform-misc/alfresco-transform-misc-boot/src/main/java/org/alfresco/{transformer => transform/misc}/Application.java (92%) rename alfresco-transform-misc/alfresco-transform-misc-boot/src/main/java/org/alfresco/{transformer => transform/misc}/MiscController.java (91%) rename alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/{transformer => transform/misc}/MiscControllerTest.java (96%) rename alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/{transformer => transform/misc}/MiscMetadataExtractsIT.java (94%) rename alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/{transformer => transform/misc}/MiscQueueTransformServiceIT.java (92%) rename alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/{transformer => transform/misc}/MiscTransformerHttpRequestTest.java (88%) rename alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/{transformer => transform/misc}/MiscTransformsIT.java (98%) rename alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/{transformer => transform/misc}/metadataExtractors/HtmlMetadataExtractor.java (98%) rename alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/{transformer => transform/misc}/metadataExtractors/RFC822MetadataExtractor.java (98%) rename alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/{transformer => transform/misc}/transformers/AppleIWorksContentTransformer.java (96%) rename alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/{transformer => transform/misc}/transformers/EMLTransformer.java (96%) rename alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/{transformer => transform/misc}/transformers/HtmlParserContentTransformer.java (96%) rename alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/{transformer => transform/misc}/transformers/OOXMLThumbnailContentTransformer.java (96%) rename alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/{transformer => transform/misc}/transformers/SelectableTransformer.java (94%) rename alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/{transformer => transform/misc}/transformers/SelectingTransformer.java (95%) rename alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/{transformer => transform/misc}/transformers/StringExtractingContentTransformer.java (96%) rename alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/{transformer => transform/misc}/transformers/TextToPdfContentTransformer.java (97%) rename alfresco-transform-misc/alfresco-transform-misc/src/test/java/org/alfresco/{transformer => transform/misc}/transformers/HtmlParserContentTransformerTest.java (96%) rename alfresco-transform-misc/alfresco-transform-misc/src/test/java/org/alfresco/{transformer => transform/misc}/transformers/TextToPdfContentTransformerTest.java (99%) create mode 100644 alfresco-transform-model/src/main/java/org/alfresco/transform/common/TransformConfigResourceReader.java rename alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/main/java/org/alfresco/{transformer => transform/pdfRenderer}/AlfrescoPdfRendererController.java (95%) rename alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/main/java/org/alfresco/{transformer => transform/pdfRenderer}/Application.java (94%) rename alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/test/java/org/alfresco/{transformer => transform/pdfRenderer}/AlfrescoPdfRendererControllerTest.java (98%) rename alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/test/java/org/alfresco/{transformer => transform/pdfRenderer}/AlfrescoPdfRendererHttpRequestTest.java (90%) rename alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/test/java/org/alfresco/{transformer => transform/pdfRenderer}/AlfrescoPdfRendererQueueTransformServiceIT.java (92%) rename alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/test/java/org/alfresco/{transformer => transform/pdfRenderer}/AlfrescoPdfRendererTransformationIT.java (95%) rename alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer/src/main/java/org/alfresco/{transformer => transform/pdfRenderer}/PdfRendererOptionsBuilder.java (97%) rename alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer/src/main/java/org/alfresco/{transformer/executors => transform/pdfRenderer/transformers}/PdfRendererCommandExecutor.java (94%) delete mode 100644 alfresco-transform-tika/alfresco-transform-tika-boot/src/main/java/org/alfresco/transformer/TikaController.java rename alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/{transformer => transform/tika}/TikaControllerTest.java (90%) rename alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/{transformer => transform/tika}/TikaHttpRequestTest.java (90%) rename alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/{transformer => transform/tika}/TikaMetadataExtractsIT.java (99%) rename alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/{transformer => transform/tika}/TikaQueueTransformServiceIT.java (95%) rename alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/{transformer => transform/tika}/TikaTransformationIT.java (98%) create mode 100644 alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/TikaTransformEngine.java rename alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/{transformer => transform/tika}/metadataExtractors/AbstractTikaMetadataExtractor.java (92%) rename alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/{transformer => transform/tika}/metadataExtractors/DWGMetadataExtractor.java (89%) rename alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/{transformer => transform/tika}/metadataExtractors/IPTCMetadataExtractor.java (91%) rename alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/{transformer => transform/tika}/metadataExtractors/MP3MetadataExtractor.java (93%) rename alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/{transformer => transform/tika}/metadataExtractors/MailMetadataExtractor.java (93%) rename alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/{transformer => transform/tika}/metadataExtractors/OfficeMetadataExtractor.java (93%) rename alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/{transformer => transform/tika}/metadataExtractors/OpenDocumentMetadataExtractor.java (95%) rename alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/{transformer => transform/tika}/metadataExtractors/PdfBoxMetadataExtractor.java (85%) rename alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/{transformer => transform/tika}/metadataExtractors/PoiMetadataExtractor.java (94%) rename alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/{transformer => transform/tika}/metadataExtractors/TikaAudioMetadataExtractor.java (93%) rename alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/{transformer => transform/tika}/metadataExtractors/TikaAutoMetadataExtractor.java (94%) rename alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/{transformer => transform}/tika/parsers/ExifToolParser.java (97%) rename alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/{transformer/executors => transform/tika/parsers}/TikaOfficeDetectParser.java (97%) create mode 100644 alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/ArchiveTransformer.java create mode 100644 alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/GenericTikaTransformer.java create mode 100644 alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/OOXMLTransformer.java create mode 100644 alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/OfficeTransformer.java create mode 100644 alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/OutlookMsgTransformer.java create mode 100644 alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/PdfBoxTransformer.java create mode 100644 alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/PoiTransformer.java create mode 100644 alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/TextMiningTransformer.java create mode 100644 alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/Tika.java create mode 100644 alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/TikaAutoTransformer.java delete mode 100644 alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/executors/Tika.java delete mode 100644 alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/executors/TikaJavaExecutor.java rename alfresco-transform-tika/alfresco-transform-tika/src/test/java/org/alfresco/{transformer => transform/tika}/metadataExtractors/IPTCMetadataExtractorTest.java (91%) rename alfresco-transform-tika/alfresco-transform-tika/src/test/java/org/alfresco/{transformer => transform}/tika/parsers/ExifToolParserTest.java (92%) rename alfresco-transform-tika/alfresco-transform-tika/src/test/java/org/alfresco/{transformer/executors/TikaJavaExecutorTest.java => transform/tika/transformers/GenericTikaTransformerTest.java} (84%) create mode 100644 t-engine-base/README.md create mode 100644 t-engine-base/pom.xml rename {alfresco-transform-tika/alfresco-transform-tika-boot/src/main/java/org/alfresco/transformer => t-engine-base/src/main/java/org/alfresco/transform/base}/Application.java (79%) create mode 100644 t-engine-base/src/main/java/org/alfresco/transform/base/CustomTransformer.java create mode 100644 t-engine-base/src/main/java/org/alfresco/transform/base/QueueTransformService.java create mode 100644 t-engine-base/src/main/java/org/alfresco/transform/base/TransformController.java create mode 100644 t-engine-base/src/main/java/org/alfresco/transform/base/TransformEngine.java create mode 100644 t-engine-base/src/main/java/org/alfresco/transform/base/TransformInterceptor.java create mode 100644 t-engine-base/src/main/java/org/alfresco/transform/base/TransformRegistryImpl.java create mode 100644 t-engine-base/src/main/java/org/alfresco/transform/base/clients/AlfrescoSharedFileStoreClient.java create mode 100644 t-engine-base/src/main/java/org/alfresco/transform/base/config/WebApplicationConfig.java create mode 100644 t-engine-base/src/main/java/org/alfresco/transform/base/executors/AbstractCommandExecutor.java create mode 100644 t-engine-base/src/main/java/org/alfresco/transform/base/executors/CommandExecutor.java create mode 100644 t-engine-base/src/main/java/org/alfresco/transform/base/executors/ExecParameterTokenizer.java create mode 100644 t-engine-base/src/main/java/org/alfresco/transform/base/executors/JavaExecutor.java create mode 100644 t-engine-base/src/main/java/org/alfresco/transform/base/executors/RuntimeExec.java create mode 100644 t-engine-base/src/main/java/org/alfresco/transform/base/executors/Transformer.java create mode 100644 t-engine-base/src/main/java/org/alfresco/transform/base/fs/FileManager.java create mode 100644 t-engine-base/src/main/java/org/alfresco/transform/base/logging/LogEntry.java create mode 100644 t-engine-base/src/main/java/org/alfresco/transform/base/logging/StandardMessages.java create mode 100644 t-engine-base/src/main/java/org/alfresco/transform/base/messaging/MessagingConfig.java create mode 100644 t-engine-base/src/main/java/org/alfresco/transform/base/messaging/MessagingInfo.java create mode 100644 t-engine-base/src/main/java/org/alfresco/transform/base/messaging/TransformMessageConverter.java create mode 100644 t-engine-base/src/main/java/org/alfresco/transform/base/messaging/TransformReplySender.java create mode 100644 t-engine-base/src/main/java/org/alfresco/transform/base/metadataExtractors/AbstractMetadataExtractor.java create mode 100644 t-engine-base/src/main/java/org/alfresco/transform/base/model/FileRefEntity.java create mode 100644 t-engine-base/src/main/java/org/alfresco/transform/base/model/FileRefResponse.java create mode 100644 t-engine-base/src/main/java/org/alfresco/transform/base/probes/ProbeTestTransform.java create mode 100644 t-engine-base/src/main/java/org/alfresco/transform/base/util/RequestParamMap.java create mode 100644 t-engine-base/src/main/java/org/alfresco/transform/base/util/Util.java create mode 100644 t-engine-base/src/main/resources/application.yaml create mode 100644 t-engine-base/src/main/resources/engine_config.json create mode 100644 t-engine-base/src/main/resources/templates/error.html create mode 100644 t-engine-base/src/main/resources/templates/log.html create mode 100644 t-engine-base/src/test/java/org/alfresco/transform/base/AbstractHttpRequestTest.java create mode 100644 t-engine-base/src/test/java/org/alfresco/transform/base/AbstractMetadataExtractsIT.java create mode 100644 t-engine-base/src/test/java/org/alfresco/transform/base/AbstractQueueTransformServiceIT.java create mode 100644 t-engine-base/src/test/java/org/alfresco/transform/base/AbstractTransformerControllerTest.java create mode 100644 t-engine-base/src/test/java/org/alfresco/transform/base/EngineClient.java create mode 100644 t-engine-base/src/test/java/org/alfresco/transform/base/QueueTransformServiceTest.java create mode 100644 t-engine-base/src/test/java/org/alfresco/transform/base/SourceTarget.java create mode 100644 t-engine-base/src/test/java/org/alfresco/transform/base/TestFileInfo.java create mode 100644 t-engine-base/src/test/resources/engine_config_complete.json create mode 100644 t-engine-base/src/test/resources/engine_config_incomplete.json create mode 100644 t-engine-base/src/test/resources/engine_config_no_transform_options.json create mode 100644 t-engine-base/src/test/resources/engine_config_with_duplicates.json diff --git a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/pom.xml b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/pom.xml index 65d30754..50838284 100644 --- a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/pom.xml +++ b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/pom.xml @@ -102,6 +102,9 @@ org.springframework.boot spring-boot-maven-plugin + + org.alfresco.transform.base.Application + org.codehaus.mojo diff --git a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/main/java/org/alfresco/transformer/AIOController.java b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/main/java/org/alfresco/transform/aio/AIOController.java similarity index 90% rename from alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/main/java/org/alfresco/transformer/AIOController.java rename to alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/main/java/org/alfresco/transform/aio/AIOController.java index cf2c81a3..93765567 100644 --- a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/main/java/org/alfresco/transformer/AIOController.java +++ b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/main/java/org/alfresco/transform/aio/AIOController.java @@ -24,11 +24,12 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.aio; import org.alfresco.transform.config.TransformConfig; import org.alfresco.transform.common.TransformException; -import org.alfresco.transformer.executors.Transformer; +import org.alfresco.transform.base.executors.Transformer; +import org.alfresco.transformer.AbstractTransformerController; import org.alfresco.transformer.probes.ProbeTestTransform; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,8 +46,8 @@ import static org.alfresco.transform.common.Mimetype.MIMETYPE_HTML; import static org.alfresco.transform.common.Mimetype.MIMETYPE_TEXT_PLAIN; import static org.alfresco.transform.config.CoreVersionDecorator.setOrClearCoreVersion; import static org.alfresco.transform.common.RequestParamMap.CONFIG_VERSION_DEFAULT; -import static org.alfresco.transformer.util.RequestParamMap.CONFIG_VERSION; -import static org.alfresco.transformer.util.RequestParamMap.SOURCE_ENCODING; +import static org.alfresco.transform.base.util.RequestParamMap.CONFIG_VERSION; +import static org.alfresco.transform.base.util.RequestParamMap.SOURCE_ENCODING; import static org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR; import static org.springframework.http.HttpStatus.OK; @@ -56,7 +57,7 @@ public class AIOController extends AbstractTransformerController private static final Logger logger = LoggerFactory.getLogger(AIOController.class); @Autowired - private AIOTransformRegistry transformRegistry; + private AIOTransformRegistry transformRegistry; @Override public String getTransformerName() diff --git a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/main/java/org/alfresco/transformer/AIOCustomConfig.java b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/main/java/org/alfresco/transform/aio/AIOCustomConfig.java similarity index 86% rename from alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/main/java/org/alfresco/transformer/AIOCustomConfig.java rename to alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/main/java/org/alfresco/transform/aio/AIOCustomConfig.java index 23fd8b85..fa1afa48 100644 --- a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/main/java/org/alfresco/transformer/AIOCustomConfig.java +++ b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/main/java/org/alfresco/transform/aio/AIOCustomConfig.java @@ -24,15 +24,16 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.aio; +import org.alfresco.transform.aio.AIOTransformRegistry; import org.alfresco.transform.registry.TransformServiceRegistry; -import org.alfresco.transformer.executors.ImageMagickCommandExecutor; -import org.alfresco.transformer.executors.LibreOfficeJavaExecutor; -import org.alfresco.transformer.executors.PdfRendererCommandExecutor; -import org.alfresco.transformer.executors.TikaJavaExecutor; -import org.alfresco.transformer.executors.Transformer; -import org.alfresco.transformer.transformers.SelectingTransformer; +import org.alfresco.transform.imagemagick.transformers.ImageMagickCommandExecutor; +import org.alfresco.transform.office.transformers.LibreOfficeJavaExecutor; +import org.alfresco.transform.pdfRenderer.transformers.PdfRendererCommandExecutor; +import org.alfresco.transform.base.executors.Transformer; +import org.alfresco.transformer.AbstractTransformerController; +import org.alfresco.transform.misc.transformers.SelectingTransformer; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -113,7 +114,7 @@ public class AIOCustomConfig List getTEnginesSortedByName() { return Stream.of(new SelectingTransformer(), - new TikaJavaExecutor(notExtractBookmarksTextDefault), +// new TikaJavaExecutor(notExtractBookmarksTextDefault), new ImageMagickCommandExecutor(imageMagickExePath, imageMagickDynPath, imageMagickRootPath, imageMagickCodersPath, imageMagickConfigPath), new LibreOfficeJavaExecutor(libreofficePath, libreofficeMaxTasksPerProcess, libreofficeTimeout, libreofficePortNumbers, libreofficeTemplateProfileDir, libreofficeIsEnabled), new PdfRendererCommandExecutor(pdfRendererPath)) diff --git a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/main/java/org/alfresco/transformer/Application.java b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/main/java/org/alfresco/transformer/Application.java deleted file mode 100644 index 0f9dc027..00000000 --- a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/main/java/org/alfresco/transformer/Application.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * #%L - * Alfresco Transform Core - * %% - * Copyright (C) 2005 - 2020 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * - - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ -package org.alfresco.transformer; - -import io.micrometer.core.instrument.MeterRegistry; -import org.alfresco.transformer.executors.ImageMagickCommandExecutor; -import org.alfresco.transformer.executors.LibreOfficeJavaExecutor; -import org.alfresco.transformer.executors.PdfRendererCommandExecutor; -import org.alfresco.transformer.executors.TikaJavaExecutor; -import org.alfresco.transformer.transformers.SelectingTransformer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.SpringApplication; -import org.springframework.boot.actuate.autoconfigure.metrics.MeterRegistryCustomizer; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; -import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.annotation.Bean; -import org.springframework.context.event.EventListener; - -import java.util.Arrays; - -import static org.alfresco.transformer.logging.StandardMessages.LICENCE; - -@SpringBootApplication -@EnableAutoConfiguration(exclude = {DataSourceAutoConfiguration.class}) -public class Application -{ - private static final Logger logger = LoggerFactory.getLogger(Application.class); - - @Value("${container.name}") - private String containerName; - - @Bean - MeterRegistryCustomizer metricsCommonTags() - { - return registry -> registry.config().commonTags("containerName", containerName); - } - - public static void main(String[] args) - { - SpringApplication.run(Application.class, args); - } - - @EventListener(ApplicationReadyEvent.class) - public void startup() - { - logger.info("--------------------------------------------------------------------------------------------------------------------------------------------------------------"); - Arrays.stream(LICENCE.split("\\n")).forEach(logger::info); - logger.info(ImageMagickCommandExecutor.LICENCE); - logger.info(LibreOfficeJavaExecutor.LICENCE); - Arrays.stream(TikaJavaExecutor.LICENCE.split("\\n")).forEach(logger::info); - logger.info(PdfRendererCommandExecutor.LICENCE); - Arrays.stream(SelectingTransformer.LICENCE.split("\\n")).forEach(logger::info); - logger.info("--------------------------------------------------------------------------------------------------------------------------------------------------------------"); - - logger.info("Starting application components... Done"); - } -} diff --git a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOControllerHttpRequestTest.java b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOControllerHttpRequestTest.java similarity index 84% rename from alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOControllerHttpRequestTest.java rename to alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOControllerHttpRequestTest.java index 037fa750..f0090402 100644 --- a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOControllerHttpRequestTest.java +++ b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOControllerHttpRequestTest.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2021 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,13 +24,14 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.aio; +import org.alfresco.transformer.AbstractHttpRequestTest; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.context.SpringBootTest.WebEnvironment; @SpringBootTest(webEnvironment = WebEnvironment.RANDOM_PORT) -public class AIOControllerHttpRequestTest extends AbstractHttpRequestTest +public class AIOControllerHttpRequestTest extends AbstractHttpRequestTest { @Override diff --git a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOControllerImageMagickTest.java b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOControllerImageMagickTest.java similarity index 86% rename from alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOControllerImageMagickTest.java rename to alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOControllerImageMagickTest.java index 04ba2954..fbdc87fd 100644 --- a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOControllerImageMagickTest.java +++ b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOControllerImageMagickTest.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2021 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,14 +24,19 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.aio; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.Map; -import org.alfresco.transformer.executors.Transformer; +import org.alfresco.transform.aio.AIOController; +import org.alfresco.transform.aio.AIOCustomConfig; +import org.alfresco.transform.aio.AIOTransformRegistry; +import org.alfresco.transform.imagemagick.ImageMagickControllerTest; +import org.alfresco.transform.base.executors.Transformer; +import org.alfresco.transformer.AbstractTransformerController; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -51,8 +56,7 @@ public class AIOControllerImageMagickTest extends ImageMagickControllerTest { // All tests contained in ImageMagickControllerTest - @Autowired - AIOTransformRegistry transformRegistry; + @Autowired AIOTransformRegistry transformRegistry; @BeforeEach @Override public void before() throws IOException @@ -67,7 +71,7 @@ public class AIOControllerImageMagickTest extends ImageMagickControllerTest } @Override - protected AbstractTransformerController getController() + protected AbstractTransformerController getController() { return controller; } diff --git a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOControllerLibreOfficeTest.java b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOControllerLibreOfficeTest.java similarity index 85% rename from alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOControllerLibreOfficeTest.java rename to alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOControllerLibreOfficeTest.java index 8bf1a2ef..1efe678a 100644 --- a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOControllerLibreOfficeTest.java +++ b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOControllerLibreOfficeTest.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2021 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,14 +24,19 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.aio; import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Map; -import org.alfresco.transformer.executors.LibreOfficeJavaExecutor; -import org.alfresco.transformer.executors.Transformer; +import org.alfresco.transform.aio.AIOController; +import org.alfresco.transform.aio.AIOCustomConfig; +import org.alfresco.transform.aio.AIOTransformRegistry; +import org.alfresco.transform.office.LibreOfficeControllerTest; +import org.alfresco.transform.office.transformers.LibreOfficeJavaExecutor; +import org.alfresco.transform.base.executors.Transformer; +import org.alfresco.transformer.AbstractTransformerController; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; @@ -56,8 +61,7 @@ public class AIOControllerLibreOfficeTest extends LibreOfficeControllerTest assertTrue(controller instanceof AIOController,"Wrong controller wired for test"); } - @Autowired - AIOTransformRegistry transformRegistry; + @Autowired AIOTransformRegistry transformRegistry; @Override // Used by the super class to mock the javaExecutor, a different implementation is required here diff --git a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOControllerMiscTest.java b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOControllerMiscTest.java similarity index 86% rename from alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOControllerMiscTest.java rename to alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOControllerMiscTest.java index 4e4be7c4..d3272d48 100644 --- a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOControllerMiscTest.java +++ b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOControllerMiscTest.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2021 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,17 +24,20 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.aio; import static org.junit.jupiter.api.Assertions.assertTrue; +import org.alfresco.transform.aio.AIOController; +import org.alfresco.transform.aio.AIOCustomConfig; +import org.alfresco.transform.misc.MiscControllerTest; import org.junit.jupiter.api.Test; import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; import org.springframework.context.annotation.Import; @WebMvcTest(AIOController.class) @Import(AIOCustomConfig.class) -public class AIOControllerMiscTest extends MiscControllerTest +public class AIOControllerMiscTest extends MiscControllerTest { //Tests contained in MiscControllerTest diff --git a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOControllerPdfRendererTest.java b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOControllerPdfRendererTest.java similarity index 87% rename from alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOControllerPdfRendererTest.java rename to alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOControllerPdfRendererTest.java index 063cb1db..fd8315cf 100644 --- a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOControllerPdfRendererTest.java +++ b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOControllerPdfRendererTest.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2021 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,14 +24,18 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.aio; import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Map; -import org.alfresco.transformer.executors.Transformer; +import org.alfresco.transform.aio.AIOController; +import org.alfresco.transform.aio.AIOCustomConfig; +import org.alfresco.transform.aio.AIOTransformRegistry; +import org.alfresco.transform.pdfRenderer.AlfrescoPdfRendererControllerTest; +import org.alfresco.transform.base.executors.Transformer; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; @@ -48,8 +52,7 @@ import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilde */ public class AIOControllerPdfRendererTest extends AlfrescoPdfRendererControllerTest { - @Autowired - AIOTransformRegistry transformRegistry; + @Autowired AIOTransformRegistry transformRegistry; @Override protected void setFields() diff --git a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOControllerTest.java b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOControllerTest.java similarity index 92% rename from alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOControllerTest.java rename to alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOControllerTest.java index f4dd759a..58794ddd 100644 --- a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOControllerTest.java +++ b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOControllerTest.java @@ -24,12 +24,15 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.aio; import java.io.IOException; +import org.alfresco.transform.aio.AIOController; +import org.alfresco.transform.aio.AIOCustomConfig; import org.alfresco.transform.client.model.TransformRequest; import org.alfresco.transform.config.TransformConfig; +import org.alfresco.transformer.AbstractTransformerController; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; diff --git a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOControllerTikaTest.java b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOControllerTikaTest.java similarity index 89% rename from alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOControllerTikaTest.java rename to alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOControllerTikaTest.java index 12e62476..403145b7 100644 --- a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOControllerTikaTest.java +++ b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOControllerTikaTest.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2021 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,10 +24,13 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.aio; import static org.junit.jupiter.api.Assertions.assertTrue; +import org.alfresco.transform.aio.AIOController; +import org.alfresco.transform.aio.AIOCustomConfig; +import org.alfresco.transform.tika.TikaControllerTest; import org.junit.jupiter.api.Test; import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; import org.springframework.context.annotation.Import; diff --git a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOCustomConfigTest.java b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOCustomConfigTest.java similarity index 90% rename from alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOCustomConfigTest.java rename to alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOCustomConfigTest.java index 2865c1df..89d47a37 100644 --- a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOCustomConfigTest.java +++ b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOCustomConfigTest.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2021 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,8 +24,10 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.aio; +import org.alfresco.transform.aio.AIOController; +import org.alfresco.transform.aio.AIOCustomConfig; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; diff --git a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOImageMagickIT.java b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOImageMagickIT.java similarity index 84% rename from alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOImageMagickIT.java rename to alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOImageMagickIT.java index d6160b6a..ff85ada1 100644 --- a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOImageMagickIT.java +++ b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOImageMagickIT.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2021 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,7 +24,9 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.aio; + +import org.alfresco.transform.imagemagick.ImageMagickTransformationIT; public class AIOImageMagickIT extends ImageMagickTransformationIT { diff --git a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOLibreOfficeTransformationIT.java b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOLibreOfficeTransformationIT.java similarity index 85% rename from alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOLibreOfficeTransformationIT.java rename to alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOLibreOfficeTransformationIT.java index ffa86390..a999d970 100644 --- a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOLibreOfficeTransformationIT.java +++ b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOLibreOfficeTransformationIT.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2021 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,7 +24,9 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.aio; + +import org.alfresco.transform.office.LibreOfficeTransformationIT; public class AIOLibreOfficeTransformationIT extends LibreOfficeTransformationIT { diff --git a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOMiscMetadataExtractsIT.java b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOMiscMetadataExtractsIT.java similarity index 89% rename from alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOMiscMetadataExtractsIT.java rename to alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOMiscMetadataExtractsIT.java index df7033b6..6b11b12a 100644 --- a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOMiscMetadataExtractsIT.java +++ b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOMiscMetadataExtractsIT.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2021 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,7 +24,9 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.aio; + +import org.alfresco.transform.misc.MiscMetadataExtractsIT; /** * Metadata integration tests in the Misc T-Engine, but run from the AIO T-Engine. diff --git a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOMiscTransformsIT.java b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOMiscTransformsIT.java similarity index 85% rename from alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOMiscTransformsIT.java rename to alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOMiscTransformsIT.java index 614cb78f..36cbfa58 100644 --- a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOMiscTransformsIT.java +++ b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOMiscTransformsIT.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2021 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,7 +24,9 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.aio; + +import org.alfresco.transform.misc.MiscTransformsIT; public class AIOMiscTransformsIT extends MiscTransformsIT { diff --git a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOPdfRendererIT.java b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOPdfRendererIT.java similarity index 84% rename from alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOPdfRendererIT.java rename to alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOPdfRendererIT.java index 2ade1494..cd440114 100644 --- a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOPdfRendererIT.java +++ b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOPdfRendererIT.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2021 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,7 +24,9 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.aio; + +import org.alfresco.transform.pdfRenderer.AlfrescoPdfRendererTransformationIT; public class AIOPdfRendererIT extends AlfrescoPdfRendererTransformationIT { // Tests are in AlfrescoPdfRendererTransformationIT diff --git a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOQueueTransformServiceIT.java b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOQueueTransformServiceIT.java similarity index 92% rename from alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOQueueTransformServiceIT.java rename to alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOQueueTransformServiceIT.java index a9c6ea6c..adebbdac 100644 --- a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOQueueTransformServiceIT.java +++ b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOQueueTransformServiceIT.java @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.aio; import static org.alfresco.transform.common.Mimetype.MIMETYPE_HTML; import static org.alfresco.transform.common.Mimetype.MIMETYPE_TEXT_PLAIN; @@ -32,6 +32,7 @@ import static org.alfresco.transform.common.Mimetype.MIMETYPE_TEXT_PLAIN; import java.util.UUID; import org.alfresco.transform.client.model.TransformRequest; +import org.alfresco.transformer.AbstractQueueTransformServiceIT; import org.springframework.boot.test.context.SpringBootTest; /** diff --git a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOTikaMetadataExtractsIT.java b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOTikaMetadataExtractsIT.java similarity index 89% rename from alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOTikaMetadataExtractsIT.java rename to alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOTikaMetadataExtractsIT.java index b0b2b83c..197478a8 100644 --- a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOTikaMetadataExtractsIT.java +++ b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOTikaMetadataExtractsIT.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2021 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,7 +24,9 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.aio; + +import org.alfresco.transform.tika.TikaMetadataExtractsIT; /** * Metadata integration tests in the Tika T-Engine, but run from the AIO T-Engine. diff --git a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOTikaTransformationIT.java b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOTikaTransformationIT.java similarity index 85% rename from alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOTikaTransformationIT.java rename to alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOTikaTransformationIT.java index 01f355b4..d4fb1657 100644 --- a/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transformer/AIOTikaTransformationIT.java +++ b/alfresco-transform-core-aio/alfresco-transform-core-aio-boot/src/test/java/org/alfresco/transform/aio/AIOTikaTransformationIT.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2021 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,7 +24,9 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.aio; + +import org.alfresco.transform.tika.TikaTransformationIT; public class AIOTikaTransformationIT extends TikaTransformationIT { diff --git a/alfresco-transform-core-aio/alfresco-transform-core-aio/src/main/java/org/alfresco/transformer/AIOTransformRegistry.java b/alfresco-transform-core-aio/alfresco-transform-core-aio/src/main/java/org/alfresco/transform/aio/AIOTransformRegistry.java similarity index 96% rename from alfresco-transform-core-aio/alfresco-transform-core-aio/src/main/java/org/alfresco/transformer/AIOTransformRegistry.java rename to alfresco-transform-core-aio/alfresco-transform-core-aio/src/main/java/org/alfresco/transform/aio/AIOTransformRegistry.java index a19fcb7a..3a92f182 100644 --- a/alfresco-transform-core-aio/alfresco-transform-core-aio/src/main/java/org/alfresco/transformer/AIOTransformRegistry.java +++ b/alfresco-transform-core-aio/alfresco-transform-core-aio/src/main/java/org/alfresco/transform/aio/AIOTransformRegistry.java @@ -24,14 +24,14 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.aio; import com.fasterxml.jackson.databind.ObjectMapper; import org.alfresco.transform.config.TransformConfig; import org.alfresco.transform.registry.AbstractTransformRegistry; import org.alfresco.transform.registry.CombinedTransformConfig; import org.alfresco.transform.registry.TransformCache; -import org.alfresco.transformer.executors.Transformer; +import org.alfresco.transform.base.executors.Transformer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/alfresco-transform-core-aio/alfresco-transform-core-aio/src/test/java/org/alfresco/transformer/AIOTransformRegistryTest.java b/alfresco-transform-core-aio/alfresco-transform-core-aio/src/test/java/org/alfresco/transform/aio/AIOTransformRegistryTest.java similarity index 96% rename from alfresco-transform-core-aio/alfresco-transform-core-aio/src/test/java/org/alfresco/transformer/AIOTransformRegistryTest.java rename to alfresco-transform-core-aio/alfresco-transform-core-aio/src/test/java/org/alfresco/transform/aio/AIOTransformRegistryTest.java index f70e1e92..855c4c3d 100644 --- a/alfresco-transform-core-aio/alfresco-transform-core-aio/src/test/java/org/alfresco/transformer/AIOTransformRegistryTest.java +++ b/alfresco-transform-core-aio/alfresco-transform-core-aio/src/test/java/org/alfresco/transform/aio/AIOTransformRegistryTest.java @@ -24,13 +24,12 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.aio; import com.fasterxml.jackson.databind.ObjectMapper; +import org.alfresco.transform.aio.AIOTransformRegistry; import org.alfresco.transform.config.TransformConfig; -import org.alfresco.transformer.executors.TikaJavaExecutor; -import org.alfresco.transformer.executors.Transformer; -import org.alfresco.transformer.transformers.SelectingTransformer; +import org.alfresco.transform.base.executors.Transformer; import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.text.PDFTextStripper; import org.junit.jupiter.api.BeforeEach; @@ -50,7 +49,7 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.alfresco.transformer.util.RequestParamMap.PAGE_LIMIT; +import static org.alfresco.transform.base.util.RequestParamMap.PAGE_LIMIT; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -67,8 +66,8 @@ public class AIOTransformRegistryTest @BeforeEach public void before() throws Exception { - aioTransformerRegistry.registerTransformer(new SelectingTransformer()); - aioTransformerRegistry.registerTransformer(new TikaJavaExecutor()); +// aioTransformerRegistry.registerTransformer(new SelectingTransformer()); +// aioTransformerRegistry.registerTransformer(new TikaJavaExecutor()); aioTransformerRegistry.registerCombinedTransformers(); } diff --git a/alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/pom.xml b/alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/pom.xml index f3628f57..4845b116 100644 --- a/alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/pom.xml +++ b/alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/pom.xml @@ -66,6 +66,9 @@ org.springframework.boot spring-boot-maven-plugin + + org.alfresco.transform.base.Application + org.codehaus.mojo diff --git a/alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/main/java/org/alfresco/transformer/Application.java b/alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/main/java/org/alfresco/transform/imagemagick/Application.java similarity index 94% rename from alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/main/java/org/alfresco/transformer/Application.java rename to alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/main/java/org/alfresco/transform/imagemagick/Application.java index 6a0f938f..d00d195f 100644 --- a/alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/main/java/org/alfresco/transformer/Application.java +++ b/alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/main/java/org/alfresco/transform/imagemagick/Application.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2020 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,10 +24,10 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.imagemagick; import io.micrometer.core.instrument.MeterRegistry; -import org.alfresco.transformer.executors.ImageMagickCommandExecutor; +import org.alfresco.transform.imagemagick.transformers.ImageMagickCommandExecutor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; diff --git a/alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/main/java/org/alfresco/transformer/ImageMagickController.java b/alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/main/java/org/alfresco/transform/imagemagick/ImageMagickController.java similarity index 96% rename from alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/main/java/org/alfresco/transformer/ImageMagickController.java rename to alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/main/java/org/alfresco/transform/imagemagick/ImageMagickController.java index 956e1712..bcfc4a86 100644 --- a/alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/main/java/org/alfresco/transformer/ImageMagickController.java +++ b/alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/main/java/org/alfresco/transform/imagemagick/ImageMagickController.java @@ -24,9 +24,10 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.imagemagick; -import org.alfresco.transformer.executors.ImageMagickCommandExecutor; +import org.alfresco.transform.imagemagick.transformers.ImageMagickCommandExecutor; +import org.alfresco.transformer.AbstractTransformerController; import org.alfresco.transformer.probes.ProbeTestTransform; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/test/java/org/alfresco/transformer/ImageMagickControllerTest.java b/alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/test/java/org/alfresco/transform/imagemagick/ImageMagickControllerTest.java similarity index 98% rename from alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/test/java/org/alfresco/transformer/ImageMagickControllerTest.java rename to alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/test/java/org/alfresco/transform/imagemagick/ImageMagickControllerTest.java index f1be9481..3f770d78 100644 --- a/alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/test/java/org/alfresco/transformer/ImageMagickControllerTest.java +++ b/alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/test/java/org/alfresco/transform/imagemagick/ImageMagickControllerTest.java @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.imagemagick; import static org.alfresco.transform.common.RequestParamMap.ENDPOINT_TRANSFORM; import static org.hamcrest.Matchers.containsString; @@ -58,7 +58,9 @@ import javax.annotation.PostConstruct; import org.alfresco.transform.client.model.TransformReply; import org.alfresco.transform.client.model.TransformRequest; -import org.alfresco.transformer.executors.ImageMagickCommandExecutor; +import org.alfresco.transform.imagemagick.transformers.ImageMagickCommandExecutor; +import org.alfresco.transformer.AbstractTransformerController; +import org.alfresco.transformer.AbstractTransformerControllerTest; import org.alfresco.transformer.executors.RuntimeExec; import org.alfresco.transformer.executors.RuntimeExec.ExecutionResult; import org.alfresco.transformer.model.FileRefEntity; diff --git a/alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/test/java/org/alfresco/transformer/ImageMagickHttpRequestTest.java b/alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/test/java/org/alfresco/transform/imagemagick/ImageMagickHttpRequestTest.java similarity index 90% rename from alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/test/java/org/alfresco/transformer/ImageMagickHttpRequestTest.java rename to alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/test/java/org/alfresco/transform/imagemagick/ImageMagickHttpRequestTest.java index 3c5082fb..c7e1c28f 100644 --- a/alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/test/java/org/alfresco/transformer/ImageMagickHttpRequestTest.java +++ b/alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/test/java/org/alfresco/transform/imagemagick/ImageMagickHttpRequestTest.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2019 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,8 +24,9 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.imagemagick; +import org.alfresco.transformer.AbstractHttpRequestTest; import org.springframework.boot.test.context.SpringBootTest; /** diff --git a/alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/test/java/org/alfresco/transformer/ImageMagickQueueTransformServiceIT.java b/alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/test/java/org/alfresco/transform/imagemagick/ImageMagickQueueTransformServiceIT.java similarity index 95% rename from alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/test/java/org/alfresco/transformer/ImageMagickQueueTransformServiceIT.java rename to alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/test/java/org/alfresco/transform/imagemagick/ImageMagickQueueTransformServiceIT.java index d415bbce..3e9d14bd 100644 --- a/alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/test/java/org/alfresco/transformer/ImageMagickQueueTransformServiceIT.java +++ b/alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/test/java/org/alfresco/transform/imagemagick/ImageMagickQueueTransformServiceIT.java @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.imagemagick; import static org.alfresco.transform.common.Mimetype.MIMETYPE_IMAGE_JPEG; import static org.alfresco.transform.common.Mimetype.MIMETYPE_IMAGE_PNG; @@ -32,6 +32,7 @@ import static org.alfresco.transform.common.Mimetype.MIMETYPE_IMAGE_PNG; import java.util.UUID; import org.alfresco.transform.client.model.TransformRequest; +import org.alfresco.transformer.AbstractQueueTransformServiceIT; import org.springframework.boot.test.context.SpringBootTest; /** diff --git a/alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/test/java/org/alfresco/transformer/ImageMagickTransformationIT.java b/alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/test/java/org/alfresco/transform/imagemagick/ImageMagickTransformationIT.java similarity index 99% rename from alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/test/java/org/alfresco/transformer/ImageMagickTransformationIT.java rename to alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/test/java/org/alfresco/transform/imagemagick/ImageMagickTransformationIT.java index b85e2ea0..d2d6e6aa 100644 --- a/alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/test/java/org/alfresco/transformer/ImageMagickTransformationIT.java +++ b/alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot/src/test/java/org/alfresco/transform/imagemagick/ImageMagickTransformationIT.java @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.imagemagick; import static java.text.MessageFormat.format; import static java.util.Collections.emptyMap; @@ -76,6 +76,7 @@ import java.util.stream.Stream; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import org.alfresco.transformer.TestFileInfo; import org.apache.commons.lang3.tuple.Pair; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; diff --git a/alfresco-transform-imagemagick/alfresco-transform-imagemagick/src/main/java/org/alfresco/transformer/ImageMagickOptionsBuilder.java b/alfresco-transform-imagemagick/alfresco-transform-imagemagick/src/main/java/org/alfresco/transform/imagemagick/ImageMagickOptionsBuilder.java similarity index 99% rename from alfresco-transform-imagemagick/alfresco-transform-imagemagick/src/main/java/org/alfresco/transformer/ImageMagickOptionsBuilder.java rename to alfresco-transform-imagemagick/alfresco-transform-imagemagick/src/main/java/org/alfresco/transform/imagemagick/ImageMagickOptionsBuilder.java index 5486071f..8270524b 100644 --- a/alfresco-transform-imagemagick/alfresco-transform-imagemagick/src/main/java/org/alfresco/transformer/ImageMagickOptionsBuilder.java +++ b/alfresco-transform-imagemagick/alfresco-transform-imagemagick/src/main/java/org/alfresco/transform/imagemagick/ImageMagickOptionsBuilder.java @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.imagemagick; import com.google.common.collect.ImmutableList; import org.alfresco.transform.common.TransformException; diff --git a/alfresco-transform-imagemagick/alfresco-transform-imagemagick/src/main/java/org/alfresco/transformer/executors/ImageMagickCommandExecutor.java b/alfresco-transform-imagemagick/alfresco-transform-imagemagick/src/main/java/org/alfresco/transform/imagemagick/transformers/ImageMagickCommandExecutor.java similarity index 97% rename from alfresco-transform-imagemagick/alfresco-transform-imagemagick/src/main/java/org/alfresco/transformer/executors/ImageMagickCommandExecutor.java rename to alfresco-transform-imagemagick/alfresco-transform-imagemagick/src/main/java/org/alfresco/transform/imagemagick/transformers/ImageMagickCommandExecutor.java index 641b50d8..dc8f3988 100644 --- a/alfresco-transform-imagemagick/alfresco-transform-imagemagick/src/main/java/org/alfresco/transformer/executors/ImageMagickCommandExecutor.java +++ b/alfresco-transform-imagemagick/alfresco-transform-imagemagick/src/main/java/org/alfresco/transform/imagemagick/transformers/ImageMagickCommandExecutor.java @@ -24,10 +24,12 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer.executors; +package org.alfresco.transform.imagemagick.transformers; +import org.alfresco.transform.imagemagick.ImageMagickOptionsBuilder; import org.alfresco.transform.common.TransformException; -import org.alfresco.transformer.ImageMagickOptionsBuilder; +import org.alfresco.transformer.executors.AbstractCommandExecutor; +import org.alfresco.transformer.executors.RuntimeExec; import java.io.File; import java.util.HashMap; diff --git a/alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/pom.xml b/alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/pom.xml index 12bc18aa..bb0e47b1 100644 --- a/alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/pom.xml +++ b/alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/pom.xml @@ -74,6 +74,9 @@ org.springframework.boot spring-boot-maven-plugin + + org.alfresco.transform.base.Application + org.codehaus.mojo diff --git a/alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/main/java/org/alfresco/transformer/Application.java b/alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/main/java/org/alfresco/transform/office/Application.java similarity index 95% rename from alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/main/java/org/alfresco/transformer/Application.java rename to alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/main/java/org/alfresco/transform/office/Application.java index 1fed620e..30d52194 100644 --- a/alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/main/java/org/alfresco/transformer/Application.java +++ b/alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/main/java/org/alfresco/transform/office/Application.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2020 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,10 +24,10 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.office; import io.micrometer.core.instrument.MeterRegistry; -import org.alfresco.transformer.executors.LibreOfficeJavaExecutor; +import org.alfresco.transform.office.transformers.LibreOfficeJavaExecutor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; diff --git a/alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/main/java/org/alfresco/transformer/LibreOfficeController.java b/alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/main/java/org/alfresco/transform/office/LibreOfficeController.java similarity index 96% rename from alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/main/java/org/alfresco/transformer/LibreOfficeController.java rename to alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/main/java/org/alfresco/transform/office/LibreOfficeController.java index 23670f5a..d07de1ee 100644 --- a/alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/main/java/org/alfresco/transformer/LibreOfficeController.java +++ b/alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/main/java/org/alfresco/transform/office/LibreOfficeController.java @@ -24,9 +24,10 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.office; -import org.alfresco.transformer.executors.LibreOfficeJavaExecutor; +import org.alfresco.transformer.AbstractTransformerController; +import org.alfresco.transform.office.transformers.LibreOfficeJavaExecutor; import org.alfresco.transformer.probes.ProbeTestTransform; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/test/java/org/alfresco/transformer/LibreOfficeControllerTest.java b/alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/test/java/org/alfresco/transform/office/LibreOfficeControllerTest.java similarity index 98% rename from alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/test/java/org/alfresco/transformer/LibreOfficeControllerTest.java rename to alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/test/java/org/alfresco/transform/office/LibreOfficeControllerTest.java index bd63ede7..57fa86f2 100644 --- a/alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/test/java/org/alfresco/transformer/LibreOfficeControllerTest.java +++ b/alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/test/java/org/alfresco/transform/office/LibreOfficeControllerTest.java @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.office; import static org.alfresco.transform.common.Mimetype.MIMETYPE_PDF; import static org.alfresco.transform.common.RequestParamMap.ENDPOINT_TRANSFORM; @@ -58,7 +58,9 @@ import javax.annotation.PostConstruct; import org.alfresco.transform.client.model.TransformReply; import org.alfresco.transform.client.model.TransformRequest; -import org.alfresco.transformer.executors.LibreOfficeJavaExecutor; +import org.alfresco.transformer.AbstractTransformerController; +import org.alfresco.transformer.AbstractTransformerControllerTest; +import org.alfresco.transform.office.transformers.LibreOfficeJavaExecutor; import org.alfresco.transformer.executors.RuntimeExec.ExecutionResult; import org.alfresco.transformer.model.FileRefEntity; import org.alfresco.transformer.model.FileRefResponse; diff --git a/alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/test/java/org/alfresco/transformer/LibreOfficeHttpRequestTest.java b/alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/test/java/org/alfresco/transform/office/LibreOfficeHttpRequestTest.java similarity index 90% rename from alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/test/java/org/alfresco/transformer/LibreOfficeHttpRequestTest.java rename to alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/test/java/org/alfresco/transform/office/LibreOfficeHttpRequestTest.java index 3bb6b0bc..d4e4a946 100644 --- a/alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/test/java/org/alfresco/transformer/LibreOfficeHttpRequestTest.java +++ b/alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/test/java/org/alfresco/transform/office/LibreOfficeHttpRequestTest.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2021 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,8 +24,9 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.office; +import org.alfresco.transformer.AbstractHttpRequestTest; import org.springframework.boot.test.context.SpringBootTest; /** diff --git a/alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/test/java/org/alfresco/transformer/LibreOfficeQueueTransformServiceIT.java b/alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/test/java/org/alfresco/transform/office/LibreOfficeQueueTransformServiceIT.java similarity index 92% rename from alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/test/java/org/alfresco/transformer/LibreOfficeQueueTransformServiceIT.java rename to alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/test/java/org/alfresco/transform/office/LibreOfficeQueueTransformServiceIT.java index e73acd9a..71bf3831 100644 --- a/alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/test/java/org/alfresco/transformer/LibreOfficeQueueTransformServiceIT.java +++ b/alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/test/java/org/alfresco/transform/office/LibreOfficeQueueTransformServiceIT.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2021 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,13 +24,14 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.office; import static org.alfresco.transform.common.Mimetype.MIMETYPE_OPENXML_WORDPROCESSING; import java.util.UUID; import org.alfresco.transform.client.model.TransformRequest; +import org.alfresco.transformer.AbstractQueueTransformServiceIT; import org.springframework.boot.test.context.SpringBootTest; /** diff --git a/alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/test/java/org/alfresco/transformer/LibreOfficeTransformationIT.java b/alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/test/java/org/alfresco/transform/office/LibreOfficeTransformationIT.java similarity index 98% rename from alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/test/java/org/alfresco/transformer/LibreOfficeTransformationIT.java rename to alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/test/java/org/alfresco/transform/office/LibreOfficeTransformationIT.java index 21c62d9b..3ecf4db4 100644 --- a/alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/test/java/org/alfresco/transformer/LibreOfficeTransformationIT.java +++ b/alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot/src/test/java/org/alfresco/transform/office/LibreOfficeTransformationIT.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2021 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.office; import static java.text.MessageFormat.format; import static java.util.function.Function.identity; @@ -54,7 +54,6 @@ import static org.alfresco.transform.common.Mimetype.MIMETYPE_XML; import static org.alfresco.transform.common.Mimetype.MIMETYPE_OPENXML_SPREADSHEET_TEMPLATE_MACRO; import static org.alfresco.transform.common.Mimetype.MIMETYPE_OPENXML_PRESENTATION_SLIDESHOW; import static org.alfresco.transform.common.Mimetype.MIMETYPE_OPENXML_PRESENTATION_SLIDESHOW_MACRO; -import static org.alfresco.transform.common.Mimetype.MIMETYPE_OUTLOOK_MSG; import static org.alfresco.transform.common.Mimetype.MIMETYPE_DITA; import static org.alfresco.transform.common.Mimetype.MIMETYPE_TEXT_PLAIN; import static org.alfresco.transform.common.Mimetype.MIMETYPE_SXI; @@ -72,6 +71,7 @@ import java.util.stream.Stream; import com.google.common.collect.ImmutableSet; +import org.alfresco.transformer.TestFileInfo; import org.apache.commons.lang3.tuple.Pair; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; diff --git a/alfresco-transform-libreoffice/alfresco-transform-libreoffice/src/main/java/org/alfresco/transformer/executors/JodConverter.java b/alfresco-transform-libreoffice/alfresco-transform-libreoffice/src/main/java/org/alfresco/transform/office/transformers/JodConverter.java similarity index 93% rename from alfresco-transform-libreoffice/alfresco-transform-libreoffice/src/main/java/org/alfresco/transformer/executors/JodConverter.java rename to alfresco-transform-libreoffice/alfresco-transform-libreoffice/src/main/java/org/alfresco/transform/office/transformers/JodConverter.java index c9623cb0..feccd272 100644 --- a/alfresco-transform-libreoffice/alfresco-transform-libreoffice/src/main/java/org/alfresco/transformer/executors/JodConverter.java +++ b/alfresco-transform-libreoffice/alfresco-transform-libreoffice/src/main/java/org/alfresco/transform/office/transformers/JodConverter.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2019 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer.executors; +package org.alfresco.transform.office.transformers; import org.artofsolving.jodconverter.office.OfficeManager; diff --git a/alfresco-transform-libreoffice/alfresco-transform-libreoffice/src/main/java/org/alfresco/transformer/executors/JodConverterSharedInstance.java b/alfresco-transform-libreoffice/alfresco-transform-libreoffice/src/main/java/org/alfresco/transform/office/transformers/JodConverterSharedInstance.java similarity index 99% rename from alfresco-transform-libreoffice/alfresco-transform-libreoffice/src/main/java/org/alfresco/transformer/executors/JodConverterSharedInstance.java rename to alfresco-transform-libreoffice/alfresco-transform-libreoffice/src/main/java/org/alfresco/transform/office/transformers/JodConverterSharedInstance.java index 8dca7a0d..4a2e7c5d 100644 --- a/alfresco-transform-libreoffice/alfresco-transform-libreoffice/src/main/java/org/alfresco/transformer/executors/JodConverterSharedInstance.java +++ b/alfresco-transform-libreoffice/alfresco-transform-libreoffice/src/main/java/org/alfresco/transform/office/transformers/JodConverterSharedInstance.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2019 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer.executors; +package org.alfresco.transform.office.transformers; import static java.util.Arrays.asList; import static java.util.Objects.requireNonNull; diff --git a/alfresco-transform-libreoffice/alfresco-transform-libreoffice/src/main/java/org/alfresco/transformer/executors/LibreOfficeExtractMetadataTask.java b/alfresco-transform-libreoffice/alfresco-transform-libreoffice/src/main/java/org/alfresco/transform/office/transformers/LibreOfficeExtractMetadataTask.java similarity index 98% rename from alfresco-transform-libreoffice/alfresco-transform-libreoffice/src/main/java/org/alfresco/transformer/executors/LibreOfficeExtractMetadataTask.java rename to alfresco-transform-libreoffice/alfresco-transform-libreoffice/src/main/java/org/alfresco/transform/office/transformers/LibreOfficeExtractMetadataTask.java index 1b944c10..99d10401 100644 --- a/alfresco-transform-libreoffice/alfresco-transform-libreoffice/src/main/java/org/alfresco/transformer/executors/LibreOfficeExtractMetadataTask.java +++ b/alfresco-transform-libreoffice/alfresco-transform-libreoffice/src/main/java/org/alfresco/transform/office/transformers/LibreOfficeExtractMetadataTask.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2020 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer.executors; +package org.alfresco.transform.office.transformers; import com.sun.star.beans.PropertyValue; import com.sun.star.beans.UnknownPropertyException; diff --git a/alfresco-transform-libreoffice/alfresco-transform-libreoffice/src/main/java/org/alfresco/transformer/executors/LibreOfficeJavaExecutor.java b/alfresco-transform-libreoffice/alfresco-transform-libreoffice/src/main/java/org/alfresco/transform/office/transformers/LibreOfficeJavaExecutor.java similarity index 99% rename from alfresco-transform-libreoffice/alfresco-transform-libreoffice/src/main/java/org/alfresco/transformer/executors/LibreOfficeJavaExecutor.java rename to alfresco-transform-libreoffice/alfresco-transform-libreoffice/src/main/java/org/alfresco/transform/office/transformers/LibreOfficeJavaExecutor.java index 783db94e..fcf8a3f4 100644 --- a/alfresco-transform-libreoffice/alfresco-transform-libreoffice/src/main/java/org/alfresco/transformer/executors/LibreOfficeJavaExecutor.java +++ b/alfresco-transform-libreoffice/alfresco-transform-libreoffice/src/main/java/org/alfresco/transform/office/transformers/LibreOfficeJavaExecutor.java @@ -24,11 +24,12 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer.executors; +package org.alfresco.transform.office.transformers; import com.fasterxml.jackson.databind.ObjectMapper; import com.sun.star.task.ErrorCodeIOException; import org.alfresco.transform.common.TransformException; +import org.alfresco.transformer.executors.JavaExecutor; import org.apache.commons.lang3.StringUtils; import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.pdmodel.PDPage; diff --git a/alfresco-transform-misc/alfresco-transform-misc-boot/pom.xml b/alfresco-transform-misc/alfresco-transform-misc-boot/pom.xml index c51d2b37..06cb1ac0 100644 --- a/alfresco-transform-misc/alfresco-transform-misc-boot/pom.xml +++ b/alfresco-transform-misc/alfresco-transform-misc-boot/pom.xml @@ -62,6 +62,9 @@ org.springframework.boot spring-boot-maven-plugin + + org.alfresco.transform.base.Application + org.codehaus.mojo diff --git a/alfresco-transform-misc/alfresco-transform-misc-boot/src/main/java/org/alfresco/transformer/Application.java b/alfresco-transform-misc/alfresco-transform-misc-boot/src/main/java/org/alfresco/transform/misc/Application.java similarity index 92% rename from alfresco-transform-misc/alfresco-transform-misc-boot/src/main/java/org/alfresco/transformer/Application.java rename to alfresco-transform-misc/alfresco-transform-misc-boot/src/main/java/org/alfresco/transform/misc/Application.java index bab9cf90..3d54833b 100644 --- a/alfresco-transform-misc/alfresco-transform-misc-boot/src/main/java/org/alfresco/transformer/Application.java +++ b/alfresco-transform-misc/alfresco-transform-misc-boot/src/main/java/org/alfresco/transform/misc/Application.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2020 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,10 +24,10 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.misc; import io.micrometer.core.instrument.MeterRegistry; -import org.alfresco.transformer.transformers.SelectingTransformer; +import org.alfresco.transform.misc.SelectingTransformer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; diff --git a/alfresco-transform-misc/alfresco-transform-misc-boot/src/main/java/org/alfresco/transformer/MiscController.java b/alfresco-transform-misc/alfresco-transform-misc-boot/src/main/java/org/alfresco/transform/misc/MiscController.java similarity index 91% rename from alfresco-transform-misc/alfresco-transform-misc-boot/src/main/java/org/alfresco/transformer/MiscController.java rename to alfresco-transform-misc/alfresco-transform-misc-boot/src/main/java/org/alfresco/transform/misc/MiscController.java index 1f63447d..cf2f330d 100644 --- a/alfresco-transform-misc/alfresco-transform-misc-boot/src/main/java/org/alfresco/transformer/MiscController.java +++ b/alfresco-transform-misc/alfresco-transform-misc-boot/src/main/java/org/alfresco/transform/misc/MiscController.java @@ -24,10 +24,11 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.misc; +import org.alfresco.transformer.AbstractTransformerController; import org.alfresco.transformer.probes.ProbeTestTransform; -import org.alfresco.transformer.transformers.SelectingTransformer; +import org.alfresco.transform.misc.transformers.SelectingTransformer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Controller; @@ -39,7 +40,6 @@ import java.util.Map; import static org.alfresco.transform.common.Mimetype.MIMETYPE_HTML; import static org.alfresco.transform.common.Mimetype.MIMETYPE_TEXT_PLAIN; import static org.alfresco.transformer.util.RequestParamMap.SOURCE_ENCODING; -import static org.alfresco.transformer.util.RequestParamMap.TRANSFORM_NAME_PARAMETER; @Controller public class MiscController extends AbstractTransformerController diff --git a/alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/transformer/MiscControllerTest.java b/alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/transform/misc/MiscControllerTest.java similarity index 96% rename from alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/transformer/MiscControllerTest.java rename to alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/transform/misc/MiscControllerTest.java index f78a8c52..5e83e721 100644 --- a/alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/transformer/MiscControllerTest.java +++ b/alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/transform/misc/MiscControllerTest.java @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.misc; import static java.nio.charset.StandardCharsets.UTF_8; import static org.alfresco.transform.common.Mimetype.MIMETYPE_HTML; @@ -48,6 +48,8 @@ import java.io.UnsupportedEncodingException; import java.nio.file.Files; import org.alfresco.transform.client.model.TransformRequest; +import org.alfresco.transformer.AbstractTransformerController; +import org.alfresco.transformer.AbstractTransformerControllerTest; import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.text.PDFTextStripper; import org.junit.jupiter.api.BeforeEach; diff --git a/alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/transformer/MiscMetadataExtractsIT.java b/alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/transform/misc/MiscMetadataExtractsIT.java similarity index 94% rename from alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/transformer/MiscMetadataExtractsIT.java rename to alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/transform/misc/MiscMetadataExtractsIT.java index 4b91d312..a9d7ab4b 100644 --- a/alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/transformer/MiscMetadataExtractsIT.java +++ b/alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/transform/misc/MiscMetadataExtractsIT.java @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.misc; import static org.alfresco.transform.common.Mimetype.MIMETYPE_HTML; import static org.alfresco.transform.common.Mimetype.MIMETYPE_RFC822; @@ -33,6 +33,8 @@ import static org.alfresco.transformer.TestFileInfo.testFile; import java.util.stream.Stream; +import org.alfresco.transformer.AbstractMetadataExtractsIT; +import org.alfresco.transformer.TestFileInfo; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; diff --git a/alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/transformer/MiscQueueTransformServiceIT.java b/alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/transform/misc/MiscQueueTransformServiceIT.java similarity index 92% rename from alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/transformer/MiscQueueTransformServiceIT.java rename to alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/transform/misc/MiscQueueTransformServiceIT.java index e2801aa1..4be40772 100644 --- a/alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/transformer/MiscQueueTransformServiceIT.java +++ b/alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/transform/misc/MiscQueueTransformServiceIT.java @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.misc; import static org.alfresco.transform.common.Mimetype.MIMETYPE_HTML; import static org.alfresco.transform.common.Mimetype.MIMETYPE_TEXT_PLAIN; @@ -32,6 +32,7 @@ import static org.alfresco.transform.common.Mimetype.MIMETYPE_TEXT_PLAIN; import java.util.UUID; import org.alfresco.transform.client.model.TransformRequest; +import org.alfresco.transformer.AbstractQueueTransformServiceIT; import org.springframework.boot.test.context.SpringBootTest; @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, diff --git a/alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/transformer/MiscTransformerHttpRequestTest.java b/alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/transform/misc/MiscTransformerHttpRequestTest.java similarity index 88% rename from alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/transformer/MiscTransformerHttpRequestTest.java rename to alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/transform/misc/MiscTransformerHttpRequestTest.java index d1380d14..e53fbfcd 100644 --- a/alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/transformer/MiscTransformerHttpRequestTest.java +++ b/alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/transform/misc/MiscTransformerHttpRequestTest.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2021 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,8 +24,9 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.misc; +import org.alfresco.transformer.AbstractHttpRequestTest; import org.springframework.boot.test.context.SpringBootTest; /** diff --git a/alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/transformer/MiscTransformsIT.java b/alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/transform/misc/MiscTransformsIT.java similarity index 98% rename from alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/transformer/MiscTransformsIT.java rename to alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/transform/misc/MiscTransformsIT.java index 93f05c63..9dcf6174 100644 --- a/alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/transformer/MiscTransformsIT.java +++ b/alfresco-transform-misc/alfresco-transform-misc-boot/src/test/java/org/alfresco/transform/misc/MiscTransformsIT.java @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.misc; import static java.text.MessageFormat.format; import static java.util.function.Function.identity; @@ -63,6 +63,8 @@ import static org.springframework.http.HttpStatus.OK; import java.util.Map; import java.util.stream.Stream; +import org.alfresco.transformer.SourceTarget; +import org.alfresco.transformer.TestFileInfo; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; import org.slf4j.Logger; diff --git a/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transformer/metadataExtractors/HtmlMetadataExtractor.java b/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transform/misc/metadataExtractors/HtmlMetadataExtractor.java similarity index 98% rename from alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transformer/metadataExtractors/HtmlMetadataExtractor.java rename to alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transform/misc/metadataExtractors/HtmlMetadataExtractor.java index 694cbd74..38b3ce50 100644 --- a/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transformer/metadataExtractors/HtmlMetadataExtractor.java +++ b/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transform/misc/metadataExtractors/HtmlMetadataExtractor.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005-2020 Alfresco Software Limited + * Copyright (C) 2005-2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -26,7 +26,7 @@ */ package org.alfresco.transformer.metadataExtractors; -import org.alfresco.transformer.transformers.SelectableTransformer; +import org.alfresco.transform.misc.transformers.SelectableTransformer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transformer/metadataExtractors/RFC822MetadataExtractor.java b/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transform/misc/metadataExtractors/RFC822MetadataExtractor.java similarity index 98% rename from alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transformer/metadataExtractors/RFC822MetadataExtractor.java rename to alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transform/misc/metadataExtractors/RFC822MetadataExtractor.java index 5f646f4e..36571830 100644 --- a/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transformer/metadataExtractors/RFC822MetadataExtractor.java +++ b/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transform/misc/metadataExtractors/RFC822MetadataExtractor.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005-2020 Alfresco Software Limited + * Copyright (C) 2005-2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -26,7 +26,7 @@ */ package org.alfresco.transformer.metadataExtractors; -import org.alfresco.transformer.transformers.SelectableTransformer; +import org.alfresco.transform.misc.transformers.SelectableTransformer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transformer/transformers/AppleIWorksContentTransformer.java b/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transform/misc/transformers/AppleIWorksContentTransformer.java similarity index 96% rename from alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transformer/transformers/AppleIWorksContentTransformer.java rename to alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transform/misc/transformers/AppleIWorksContentTransformer.java index bcace46e..96a0aeab 100644 --- a/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transformer/transformers/AppleIWorksContentTransformer.java +++ b/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transform/misc/transformers/AppleIWorksContentTransformer.java @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer.transformers; +package org.alfresco.transform.misc.transformers; import com.google.common.collect.ImmutableList; import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; diff --git a/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transformer/transformers/EMLTransformer.java b/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transform/misc/transformers/EMLTransformer.java similarity index 96% rename from alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transformer/transformers/EMLTransformer.java rename to alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transform/misc/transformers/EMLTransformer.java index ed4c642f..5ae2f38b 100644 --- a/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transformer/transformers/EMLTransformer.java +++ b/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transform/misc/transformers/EMLTransformer.java @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer.transformers; +package org.alfresco.transform.misc.transformers; import org.alfresco.transformer.fs.FileManager; import org.slf4j.Logger; diff --git a/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transformer/transformers/HtmlParserContentTransformer.java b/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transform/misc/transformers/HtmlParserContentTransformer.java similarity index 96% rename from alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transformer/transformers/HtmlParserContentTransformer.java rename to alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transform/misc/transformers/HtmlParserContentTransformer.java index 857ea39f..9b3ebbba 100644 --- a/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transformer/transformers/HtmlParserContentTransformer.java +++ b/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transform/misc/transformers/HtmlParserContentTransformer.java @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer.transformers; +package org.alfresco.transform.misc.transformers; import org.htmlparser.Parser; import org.htmlparser.beans.StringBean; diff --git a/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transformer/transformers/OOXMLThumbnailContentTransformer.java b/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transform/misc/transformers/OOXMLThumbnailContentTransformer.java similarity index 96% rename from alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transformer/transformers/OOXMLThumbnailContentTransformer.java rename to alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transform/misc/transformers/OOXMLThumbnailContentTransformer.java index baed9694..d684edda 100644 --- a/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transformer/transformers/OOXMLThumbnailContentTransformer.java +++ b/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transform/misc/transformers/OOXMLThumbnailContentTransformer.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2020 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer.transformers; +package org.alfresco.transform.misc.transformers; import org.apache.poi.openxml4j.opc.OPCPackage; import org.apache.poi.openxml4j.opc.PackagePart; diff --git a/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transformer/transformers/SelectableTransformer.java b/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transform/misc/transformers/SelectableTransformer.java similarity index 94% rename from alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transformer/transformers/SelectableTransformer.java rename to alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transform/misc/transformers/SelectableTransformer.java index af89da91..0892e961 100644 --- a/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transformer/transformers/SelectableTransformer.java +++ b/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transform/misc/transformers/SelectableTransformer.java @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer.transformers; +package org.alfresco.transform.misc.transformers; import java.io.File; import java.util.Map; diff --git a/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transformer/transformers/SelectingTransformer.java b/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transform/misc/transformers/SelectingTransformer.java similarity index 95% rename from alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transformer/transformers/SelectingTransformer.java rename to alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transform/misc/transformers/SelectingTransformer.java index 67652e5f..a0f4ab67 100644 --- a/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transformer/transformers/SelectingTransformer.java +++ b/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transform/misc/transformers/SelectingTransformer.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2020 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer.transformers; +package org.alfresco.transform.misc.transformers; import com.google.common.collect.ImmutableMap; import org.alfresco.transformer.executors.Transformer; diff --git a/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transformer/transformers/StringExtractingContentTransformer.java b/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transform/misc/transformers/StringExtractingContentTransformer.java similarity index 96% rename from alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transformer/transformers/StringExtractingContentTransformer.java rename to alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transform/misc/transformers/StringExtractingContentTransformer.java index 3364b631..2aa58bad 100644 --- a/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transformer/transformers/StringExtractingContentTransformer.java +++ b/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transform/misc/transformers/StringExtractingContentTransformer.java @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer.transformers; +package org.alfresco.transform.misc.transformers; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transformer/transformers/TextToPdfContentTransformer.java b/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transform/misc/transformers/TextToPdfContentTransformer.java similarity index 97% rename from alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transformer/transformers/TextToPdfContentTransformer.java rename to alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transform/misc/transformers/TextToPdfContentTransformer.java index 4be864d6..f8fac5b5 100644 --- a/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transformer/transformers/TextToPdfContentTransformer.java +++ b/alfresco-transform-misc/alfresco-transform-misc/src/main/java/org/alfresco/transform/misc/transformers/TextToPdfContentTransformer.java @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer.transformers; +package org.alfresco.transform.misc.transformers; import org.alfresco.transformer.util.RequestParamMap; import org.apache.pdfbox.pdmodel.PDDocument; diff --git a/alfresco-transform-misc/alfresco-transform-misc/src/test/java/org/alfresco/transformer/transformers/HtmlParserContentTransformerTest.java b/alfresco-transform-misc/alfresco-transform-misc/src/test/java/org/alfresco/transform/misc/transformers/HtmlParserContentTransformerTest.java similarity index 96% rename from alfresco-transform-misc/alfresco-transform-misc/src/test/java/org/alfresco/transformer/transformers/HtmlParserContentTransformerTest.java rename to alfresco-transform-misc/alfresco-transform-misc/src/test/java/org/alfresco/transform/misc/transformers/HtmlParserContentTransformerTest.java index 3e663972..7a275886 100644 --- a/alfresco-transform-misc/alfresco-transform-misc/src/test/java/org/alfresco/transformer/transformers/HtmlParserContentTransformerTest.java +++ b/alfresco-transform-misc/alfresco-transform-misc/src/test/java/org/alfresco/transform/misc/transformers/HtmlParserContentTransformerTest.java @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer.transformers; +package org.alfresco.transform.misc.transformers; import org.junit.jupiter.api.Test; diff --git a/alfresco-transform-misc/alfresco-transform-misc/src/test/java/org/alfresco/transformer/transformers/TextToPdfContentTransformerTest.java b/alfresco-transform-misc/alfresco-transform-misc/src/test/java/org/alfresco/transform/misc/transformers/TextToPdfContentTransformerTest.java similarity index 99% rename from alfresco-transform-misc/alfresco-transform-misc/src/test/java/org/alfresco/transformer/transformers/TextToPdfContentTransformerTest.java rename to alfresco-transform-misc/alfresco-transform-misc/src/test/java/org/alfresco/transform/misc/transformers/TextToPdfContentTransformerTest.java index e80f5569..43ff8b1e 100644 --- a/alfresco-transform-misc/alfresco-transform-misc/src/test/java/org/alfresco/transformer/transformers/TextToPdfContentTransformerTest.java +++ b/alfresco-transform-misc/alfresco-transform-misc/src/test/java/org/alfresco/transform/misc/transformers/TextToPdfContentTransformerTest.java @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer.transformers; +package org.alfresco.transform.misc.transformers; import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.text.PDFTextStripper; diff --git a/alfresco-transform-model/src/main/java/org/alfresco/transform/common/TransformConfigResourceReader.java b/alfresco-transform-model/src/main/java/org/alfresco/transform/common/TransformConfigResourceReader.java new file mode 100644 index 00000000..0be7fcdf --- /dev/null +++ b/alfresco-transform-model/src/main/java/org/alfresco/transform/common/TransformConfigResourceReader.java @@ -0,0 +1,63 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.common; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.alfresco.transform.config.TransformConfig; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.Resource; +import org.springframework.core.io.ResourceLoader; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR; + +@Component +public class TransformConfigResourceReader +{ + @Autowired ResourceLoader resourceLoader; + + private ObjectMapper jsonObjectMapper = new ObjectMapper(); + + public TransformConfig read(String engineConfigLocation) + { + Resource engineConfig = resourceLoader.getResource(engineConfigLocation); + try (Reader reader = new InputStreamReader(engineConfig.getInputStream(), UTF_8)) + { + TransformConfig transformConfig = jsonObjectMapper.readValue(reader, TransformConfig.class); + return transformConfig; + } + catch (IOException e) + { + throw new TransformException(INTERNAL_SERVER_ERROR.value(), "Could not read " + engineConfigLocation, e); + } + } +} diff --git a/alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/pom.xml b/alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/pom.xml index 9fb05667..e79d5919 100644 --- a/alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/pom.xml +++ b/alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/pom.xml @@ -62,6 +62,9 @@ org.springframework.boot spring-boot-maven-plugin + + org.alfresco.transform.base.Application + org.codehaus.mojo diff --git a/alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/main/java/org/alfresco/transformer/AlfrescoPdfRendererController.java b/alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/main/java/org/alfresco/transform/pdfRenderer/AlfrescoPdfRendererController.java similarity index 95% rename from alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/main/java/org/alfresco/transformer/AlfrescoPdfRendererController.java rename to alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/main/java/org/alfresco/transform/pdfRenderer/AlfrescoPdfRendererController.java index d0e89e49..83c22d29 100644 --- a/alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/main/java/org/alfresco/transformer/AlfrescoPdfRendererController.java +++ b/alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/main/java/org/alfresco/transform/pdfRenderer/AlfrescoPdfRendererController.java @@ -24,9 +24,10 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.pdfRenderer; -import org.alfresco.transformer.executors.PdfRendererCommandExecutor; +import org.alfresco.transformer.AbstractTransformerController; +import org.alfresco.transform.pdfRenderer.transformers.PdfRendererCommandExecutor; import org.alfresco.transformer.probes.ProbeTestTransform; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/main/java/org/alfresco/transformer/Application.java b/alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/main/java/org/alfresco/transform/pdfRenderer/Application.java similarity index 94% rename from alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/main/java/org/alfresco/transformer/Application.java rename to alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/main/java/org/alfresco/transform/pdfRenderer/Application.java index 5bbf3afb..5a990709 100644 --- a/alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/main/java/org/alfresco/transformer/Application.java +++ b/alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/main/java/org/alfresco/transform/pdfRenderer/Application.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2020 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,10 +24,10 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.pdfRenderer; import io.micrometer.core.instrument.MeterRegistry; -import org.alfresco.transformer.executors.PdfRendererCommandExecutor; +import org.alfresco.transform.pdfRenderer.transformers.PdfRendererCommandExecutor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; diff --git a/alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/test/java/org/alfresco/transformer/AlfrescoPdfRendererControllerTest.java b/alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/test/java/org/alfresco/transform/pdfRenderer/AlfrescoPdfRendererControllerTest.java similarity index 98% rename from alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/test/java/org/alfresco/transformer/AlfrescoPdfRendererControllerTest.java rename to alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/test/java/org/alfresco/transform/pdfRenderer/AlfrescoPdfRendererControllerTest.java index cdea0f40..5ff09221 100644 --- a/alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/test/java/org/alfresco/transformer/AlfrescoPdfRendererControllerTest.java +++ b/alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/test/java/org/alfresco/transform/pdfRenderer/AlfrescoPdfRendererControllerTest.java @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.pdfRenderer; import static org.alfresco.transform.common.RequestParamMap.ENDPOINT_TRANSFORM; import static org.hamcrest.Matchers.containsString; @@ -59,7 +59,9 @@ import javax.annotation.PostConstruct; import org.alfresco.transform.client.model.TransformReply; import org.alfresco.transform.client.model.TransformRequest; -import org.alfresco.transformer.executors.PdfRendererCommandExecutor; +import org.alfresco.transformer.AbstractTransformerController; +import org.alfresco.transformer.AbstractTransformerControllerTest; +import org.alfresco.transform.pdfRenderer.transformers.PdfRendererCommandExecutor; import org.alfresco.transformer.executors.RuntimeExec; import org.alfresco.transformer.executors.RuntimeExec.ExecutionResult; import org.alfresco.transformer.model.FileRefEntity; diff --git a/alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/test/java/org/alfresco/transformer/AlfrescoPdfRendererHttpRequestTest.java b/alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/test/java/org/alfresco/transform/pdfRenderer/AlfrescoPdfRendererHttpRequestTest.java similarity index 90% rename from alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/test/java/org/alfresco/transformer/AlfrescoPdfRendererHttpRequestTest.java rename to alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/test/java/org/alfresco/transform/pdfRenderer/AlfrescoPdfRendererHttpRequestTest.java index caee39a3..963040c8 100644 --- a/alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/test/java/org/alfresco/transformer/AlfrescoPdfRendererHttpRequestTest.java +++ b/alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/test/java/org/alfresco/transform/pdfRenderer/AlfrescoPdfRendererHttpRequestTest.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2021 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,8 +24,9 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.pdfRenderer; +import org.alfresco.transformer.AbstractHttpRequestTest; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.context.SpringBootTest.WebEnvironment; diff --git a/alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/test/java/org/alfresco/transformer/AlfrescoPdfRendererQueueTransformServiceIT.java b/alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/test/java/org/alfresco/transform/pdfRenderer/AlfrescoPdfRendererQueueTransformServiceIT.java similarity index 92% rename from alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/test/java/org/alfresco/transformer/AlfrescoPdfRendererQueueTransformServiceIT.java rename to alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/test/java/org/alfresco/transform/pdfRenderer/AlfrescoPdfRendererQueueTransformServiceIT.java index 0e3ea58c..c956f643 100644 --- a/alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/test/java/org/alfresco/transformer/AlfrescoPdfRendererQueueTransformServiceIT.java +++ b/alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/test/java/org/alfresco/transform/pdfRenderer/AlfrescoPdfRendererQueueTransformServiceIT.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2021 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.pdfRenderer; import static org.alfresco.transform.common.Mimetype.MIMETYPE_OPENXML_WORDPROCESSING; import static org.alfresco.transform.common.Mimetype.MIMETYPE_PDF; @@ -32,6 +32,7 @@ import static org.alfresco.transform.common.Mimetype.MIMETYPE_PDF; import java.util.UUID; import org.alfresco.transform.client.model.TransformRequest; +import org.alfresco.transformer.AbstractQueueTransformServiceIT; import org.springframework.boot.test.context.SpringBootTest; /** diff --git a/alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/test/java/org/alfresco/transformer/AlfrescoPdfRendererTransformationIT.java b/alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/test/java/org/alfresco/transform/pdfRenderer/AlfrescoPdfRendererTransformationIT.java similarity index 95% rename from alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/test/java/org/alfresco/transformer/AlfrescoPdfRendererTransformationIT.java rename to alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/test/java/org/alfresco/transform/pdfRenderer/AlfrescoPdfRendererTransformationIT.java index beaf5170..613a0dc3 100644 --- a/alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/test/java/org/alfresco/transformer/AlfrescoPdfRendererTransformationIT.java +++ b/alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot/src/test/java/org/alfresco/transform/pdfRenderer/AlfrescoPdfRendererTransformationIT.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2021 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.pdfRenderer; import static java.text.MessageFormat.format; import static java.util.function.Function.identity; @@ -38,6 +38,7 @@ import static org.springframework.http.HttpStatus.OK; import java.util.Map; import java.util.stream.Stream; +import org.alfresco.transformer.TestFileInfo; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; import org.slf4j.Logger; diff --git a/alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer/src/main/java/org/alfresco/transformer/PdfRendererOptionsBuilder.java b/alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer/src/main/java/org/alfresco/transform/pdfRenderer/PdfRendererOptionsBuilder.java similarity index 97% rename from alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer/src/main/java/org/alfresco/transformer/PdfRendererOptionsBuilder.java rename to alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer/src/main/java/org/alfresco/transform/pdfRenderer/PdfRendererOptionsBuilder.java index be8d0118..fe8dea83 100644 --- a/alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer/src/main/java/org/alfresco/transformer/PdfRendererOptionsBuilder.java +++ b/alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer/src/main/java/org/alfresco/transform/pdfRenderer/PdfRendererOptionsBuilder.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2019 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.pdfRenderer; import static org.alfresco.transformer.util.Util.stringToBoolean; import static org.alfresco.transformer.util.Util.stringToInteger; diff --git a/alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer/src/main/java/org/alfresco/transformer/executors/PdfRendererCommandExecutor.java b/alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer/src/main/java/org/alfresco/transform/pdfRenderer/transformers/PdfRendererCommandExecutor.java similarity index 94% rename from alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer/src/main/java/org/alfresco/transformer/executors/PdfRendererCommandExecutor.java rename to alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer/src/main/java/org/alfresco/transform/pdfRenderer/transformers/PdfRendererCommandExecutor.java index 7c234742..cd2edbef 100644 --- a/alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer/src/main/java/org/alfresco/transformer/executors/PdfRendererCommandExecutor.java +++ b/alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer/src/main/java/org/alfresco/transform/pdfRenderer/transformers/PdfRendererCommandExecutor.java @@ -24,10 +24,12 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer.executors; +package org.alfresco.transform.pdfRenderer.transformers; import org.alfresco.transform.common.TransformException; -import org.alfresco.transformer.PdfRendererOptionsBuilder; +import org.alfresco.transform.pdfRenderer.PdfRendererOptionsBuilder; +import org.alfresco.transformer.executors.AbstractCommandExecutor; +import org.alfresco.transformer.executors.RuntimeExec; import java.io.File; import java.util.HashMap; diff --git a/alfresco-transform-tika/alfresco-transform-tika-boot/pom.xml b/alfresco-transform-tika/alfresco-transform-tika-boot/pom.xml index 575e538f..5f62c9f0 100644 --- a/alfresco-transform-tika/alfresco-transform-tika-boot/pom.xml +++ b/alfresco-transform-tika/alfresco-transform-tika-boot/pom.xml @@ -20,12 +20,12 @@ org.alfresco - alfresco-transformer-base + alfresco-t-engine-base ${project.version} org.alfresco - alfresco-transformer-base + alfresco-t-engine-base ${project.version} tests test-jar @@ -146,6 +146,9 @@ org.springframework.boot spring-boot-maven-plugin + + org.alfresco.transform.base.Application + org.codehaus.mojo diff --git a/alfresco-transform-tika/alfresco-transform-tika-boot/src/main/java/org/alfresco/transformer/TikaController.java b/alfresco-transform-tika/alfresco-transform-tika-boot/src/main/java/org/alfresco/transformer/TikaController.java deleted file mode 100644 index 4b76caa8..00000000 --- a/alfresco-transform-tika/alfresco-transform-tika-boot/src/main/java/org/alfresco/transformer/TikaController.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * #%L - * Alfresco Transform Core - * %% - * Copyright (C) 2005 - 2022 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * - - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ -package org.alfresco.transformer; - -import org.alfresco.transformer.executors.TikaJavaExecutor; -import org.alfresco.transformer.probes.ProbeTestTransform; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Controller; - -import java.io.File; -import java.util.HashMap; -import java.util.Map; - -import static org.alfresco.transformer.executors.Tika.PDF_BOX; -import static org.alfresco.transform.common.Mimetype.MIMETYPE_PDF; -import static org.alfresco.transform.common.Mimetype.MIMETYPE_TEXT_PLAIN; - -/** - * Controller for the Docker based Tika transformers. - * - * Status Codes: - * - * 200 Success - * 400 Bad Request: Invalid target mimetype - * 400 Bad Request: Request parameter is missing (missing mandatory parameter) - * 400 Bad Request: Request parameter is of the wrong type - * 400 Bad Request: Transformer exit code was not 0 (possible problem with the source file) - * 400 Bad Request: The source filename was not supplied - * 500 Internal Server Error: (no message with low level IO problems) - * 500 Internal Server Error: The target filename was not supplied (should not happen as targetExtension is checked) - * 500 Internal Server Error: Transformer version check exit code was not 0 - * 500 Internal Server Error: Transformer version check failed to create any output - * 500 Internal Server Error: Could not read the target file - * 500 Internal Server Error: The target filename was malformed (should not happen because of other checks) - * 500 Internal Server Error: Transformer failed to create an output file (the exit code was 0, so there should be some content) - * 500 Internal Server Error: Filename encoding error - * 507 Insufficient Storage: Failed to store the source file - */ -@Controller -public class TikaController extends AbstractTransformerController -{ - private static final Logger logger = LoggerFactory.getLogger(TikaController.class); - - private TikaJavaExecutor javaExecutor; - - public TikaController(@Value("${transform.core.tika.pdfBox.notExtractBookmarksTextDefault:false}") boolean notExtractBookmarksTextDefault) - { - javaExecutor= new TikaJavaExecutor(notExtractBookmarksTextDefault); - } - - @Override - public String getTransformerName() - { - return "Tika"; - } - - @Override - public String version() - { - return "Tika available"; - } - - @Override - public ProbeTestTransform getProbeTestTransform() - { - // See the Javadoc on this method and Probes.md for the choice of these values. - // the livenessPercentage is a little large as Tika does tend to suffer from slow transforms that class with a gc. - return new ProbeTestTransform(this, "quick.pdf", "quick.txt", - 60, 16, 400, 10240, 60 * 30 + 1, 60 * 15 + 20) - { - @Override - protected void executeTransformCommand(File sourceFile, File targetFile) - { - transformImpl(PDF_BOX, MIMETYPE_PDF, MIMETYPE_TEXT_PLAIN, new HashMap<>(), sourceFile, targetFile); - } - }; - } - - @Override - public void transformImpl(String transformName, String sourceMimetype, String targetMimetype, - Map transformOptions, File sourceFile, File targetFile) - { - javaExecutor.transformExtractOrEmbed(transformName, sourceMimetype, targetMimetype, transformOptions, sourceFile, targetFile); - } -} diff --git a/alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transformer/TikaControllerTest.java b/alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transform/tika/TikaControllerTest.java similarity index 90% rename from alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transformer/TikaControllerTest.java rename to alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transform/tika/TikaControllerTest.java index 81a02c7c..367261a7 100644 --- a/alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transformer/TikaControllerTest.java +++ b/alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transform/tika/TikaControllerTest.java @@ -24,30 +24,30 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.tika; import static java.nio.file.Files.readAllBytes; import static org.alfresco.transform.common.RequestParamMap.ENDPOINT_TRANSFORM; -import static org.alfresco.transformer.executors.Tika.ARCHIVE; -import static org.alfresco.transformer.executors.Tika.CSV; -import static org.alfresco.transformer.executors.Tika.DOC; -import static org.alfresco.transformer.executors.Tika.DOCX; -import static org.alfresco.transformer.executors.Tika.HTML; -import static org.alfresco.transformer.executors.Tika.MSG; -import static org.alfresco.transformer.executors.Tika.OUTLOOK_MSG; -import static org.alfresco.transformer.executors.Tika.PDF; -import static org.alfresco.transformer.executors.Tika.PDF_BOX; -import static org.alfresco.transformer.executors.Tika.POI; -import static org.alfresco.transformer.executors.Tika.POI_OFFICE; -import static org.alfresco.transformer.executors.Tika.POI_OO_XML; -import static org.alfresco.transformer.executors.Tika.PPTX; -import static org.alfresco.transformer.executors.Tika.TEXT_MINING; -import static org.alfresco.transformer.executors.Tika.TIKA_AUTO; -import static org.alfresco.transformer.executors.Tika.TXT; -import static org.alfresco.transformer.executors.Tika.XHTML; -import static org.alfresco.transformer.executors.Tika.XML; -import static org.alfresco.transformer.executors.Tika.XSLX; -import static org.alfresco.transformer.executors.Tika.ZIP; +import static org.alfresco.transform.tika.transformers.Tika.ARCHIVE; +import static org.alfresco.transform.tika.transformers.Tika.CSV; +import static org.alfresco.transform.tika.transformers.Tika.DOC; +import static org.alfresco.transform.tika.transformers.Tika.DOCX; +import static org.alfresco.transform.tika.transformers.Tika.HTML; +import static org.alfresco.transform.tika.transformers.Tika.MSG; +import static org.alfresco.transform.tika.transformers.Tika.OUTLOOK_MSG; +import static org.alfresco.transform.tika.transformers.Tika.PDF; +import static org.alfresco.transform.tika.transformers.Tika.PDF_BOX; +import static org.alfresco.transform.tika.transformers.Tika.POI; +import static org.alfresco.transform.tika.transformers.Tika.OFFICE; +import static org.alfresco.transform.tika.transformers.Tika.OOXML; +import static org.alfresco.transform.tika.transformers.Tika.PPTX; +import static org.alfresco.transform.tika.transformers.Tika.TEXT_MINING; +import static org.alfresco.transform.tika.transformers.Tika.TIKA_AUTO; +import static org.alfresco.transform.tika.transformers.Tika.TXT; +import static org.alfresco.transform.tika.transformers.Tika.XHTML; +import static org.alfresco.transform.tika.transformers.Tika.XML; +import static org.alfresco.transform.tika.transformers.Tika.XSLX; +import static org.alfresco.transform.tika.transformers.Tika.ZIP; import static org.alfresco.transform.common.Mimetype.MIMETYPE_HTML; import static org.alfresco.transform.common.Mimetype.MIMETYPE_METADATA_EMBED; import static org.alfresco.transform.common.Mimetype.MIMETYPE_OPENXML_PRESENTATION; @@ -61,8 +61,8 @@ import static org.alfresco.transform.common.Mimetype.MIMETYPE_WORD; import static org.alfresco.transform.common.Mimetype.MIMETYPE_XHTML; import static org.alfresco.transform.common.Mimetype.MIMETYPE_XML; import static org.alfresco.transform.common.Mimetype.MIMETYPE_ZIP; -import static org.alfresco.transformer.util.RequestParamMap.INCLUDE_CONTENTS; -import static org.alfresco.transformer.util.RequestParamMap.NOT_EXTRACT_BOOKMARKS_TEXT; +import static org.alfresco.transform.base.util.RequestParamMap.INCLUDE_CONTENTS; +import static org.alfresco.transform.base.util.RequestParamMap.NOT_EXTRACT_BOOKMARKS_TEXT; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -91,12 +91,14 @@ import java.util.UUID; import javax.servlet.http.HttpServletRequest; +import org.alfresco.transform.base.AbstractTransformerControllerTest; +import org.alfresco.transform.base.TransformController; import org.alfresco.transform.client.model.TransformReply; import org.alfresco.transform.client.model.TransformRequest; -import org.alfresco.transformer.executors.RuntimeExec; -import org.alfresco.transformer.model.FileRefEntity; -import org.alfresco.transformer.model.FileRefResponse; -import org.alfresco.transformer.probes.ProbeTestTransform; +import org.alfresco.transform.base.executors.RuntimeExec; +import org.alfresco.transform.base.model.FileRefEntity; +import org.alfresco.transform.base.model.FileRefResponse; +import org.alfresco.transform.base.probes.ProbeTestTransform; import org.apache.poi.ooxml.POIXMLProperties; import org.apache.poi.xssf.usermodel.XSSFWorkbook; import org.junit.jupiter.api.BeforeEach; @@ -142,9 +144,6 @@ public class TikaControllerTest extends AbstractTransformerControllerTest @Mock private RuntimeExec mockCheckCommand; - @Autowired - protected AbstractTransformerController controller; - private String targetEncoding = "UTF-8"; private String targetMimetype = MIMETYPE_TEXT_PLAIN; @@ -236,12 +235,6 @@ public class TikaControllerTest extends AbstractTransformerControllerTest when(mockExecutionResult.getStdOut()).thenReturn("STDOUT"); } - @Override - protected AbstractTransformerController getController() - { - return controller; - } - private void transform(String transform, String sourceExtension, String targetExtension, String sourceMimetype, String targetMimetype, Boolean includeContents, String expectedContentContains) throws Exception @@ -284,9 +277,10 @@ public class TikaControllerTest extends AbstractTransformerControllerTest public void testImmutableEmptyMap() { // See ACS-373 - ProbeTestTransform probeTestTransform = getController().getProbeTestTransform(); + TransformController controller = getController(); + ProbeTestTransform probeTestTransform = getProbeTestTransform(); ReflectionTestUtils.setField(probeTestTransform, "livenessTransformEnabled", true); - probeTestTransform.doTransformOrNothing(httpServletRequest, true); + probeTestTransform.doTransformOrNothing(httpServletRequest, true, controller); } @Test @@ -483,14 +477,14 @@ public class TikaControllerTest extends AbstractTransformerControllerTest @Test public void msgToTxtOfficeTest() throws Exception { - transform(POI_OFFICE, MSG, TXT, MIMETYPE_OUTLOOK_MSG, MIMETYPE_TEXT_PLAIN, null, + transform(OFFICE, MSG, TXT, MIMETYPE_OUTLOOK_MSG, MIMETYPE_TEXT_PLAIN, null, EXPECTED_MSG_CONTENT_CONTAINS); } @Test public void docToTxtOfficeTest() throws Exception { - transform(POI_OFFICE, DOC, TXT, MIMETYPE_WORD, MIMETYPE_TEXT_PLAIN, null, + transform(OFFICE, DOC, TXT, MIMETYPE_WORD, MIMETYPE_TEXT_PLAIN, null, EXPECTED_TEXT_CONTENT_CONTAINS); } @@ -508,14 +502,14 @@ public class TikaControllerTest extends AbstractTransformerControllerTest @Test public void docxToTxtOoXmlTest() throws Exception { - transform(POI_OO_XML, DOCX, TXT, MIMETYPE_OPENXML_WORDPROCESSING, MIMETYPE_TEXT_PLAIN, null, + transform(OOXML, DOCX, TXT, MIMETYPE_OPENXML_WORDPROCESSING, MIMETYPE_TEXT_PLAIN, null, EXPECTED_TEXT_CONTENT_CONTAINS); } @Test public void pptxToTxtOoXmlTest() throws Exception { - transform(POI_OO_XML, PPTX, TXT, MIMETYPE_OPENXML_PRESENTATION, MIMETYPE_TEXT_PLAIN, null, + transform(OOXML, PPTX, TXT, MIMETYPE_OPENXML_PRESENTATION, MIMETYPE_TEXT_PLAIN, null, EXPECTED_TEXT_CONTENT_CONTAINS); } diff --git a/alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transformer/TikaHttpRequestTest.java b/alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transform/tika/TikaHttpRequestTest.java similarity index 90% rename from alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transformer/TikaHttpRequestTest.java rename to alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transform/tika/TikaHttpRequestTest.java index 56aef563..360c1385 100644 --- a/alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transformer/TikaHttpRequestTest.java +++ b/alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transform/tika/TikaHttpRequestTest.java @@ -24,15 +24,11 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; - -import static org.springframework.http.MediaType.MULTIPART_FORM_DATA; +package org.alfresco.transform.tika; +import org.alfresco.transform.base.AbstractHttpRequestTest; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.context.SpringBootTest.WebEnvironment; -import org.springframework.core.io.ClassPathResource; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpHeaders; import org.springframework.util.LinkedMultiValueMap; /** diff --git a/alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transformer/TikaMetadataExtractsIT.java b/alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transform/tika/TikaMetadataExtractsIT.java similarity index 99% rename from alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transformer/TikaMetadataExtractsIT.java rename to alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transform/tika/TikaMetadataExtractsIT.java index dc7e116f..a54aa9fd 100644 --- a/alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transformer/TikaMetadataExtractsIT.java +++ b/alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transform/tika/TikaMetadataExtractsIT.java @@ -24,11 +24,11 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.tika; import static org.alfresco.transform.common.Mimetype.MIMETYPE_APP_DWG; import static org.alfresco.transform.common.Mimetype.MIMETYPE_OUTLOOK_MSG; -import static org.alfresco.transformer.TestFileInfo.testFile; +import static org.alfresco.transform.base.TestFileInfo.testFile; import static org.alfresco.transform.common.Mimetype.MIMETYPE_AUDIO_MP4; import static org.alfresco.transform.common.Mimetype.MIMETYPE_EXCEL; import static org.alfresco.transform.common.Mimetype.MIMETYPE_IMAGE_BMP; @@ -74,6 +74,8 @@ import static org.alfresco.transform.common.Mimetype.MIMETYPE_IMAGE_RAW_NEF; import java.util.stream.Stream; +import org.alfresco.transform.base.AbstractMetadataExtractsIT; +import org.alfresco.transform.base.TestFileInfo; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; diff --git a/alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transformer/TikaQueueTransformServiceIT.java b/alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transform/tika/TikaQueueTransformServiceIT.java similarity index 95% rename from alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transformer/TikaQueueTransformServiceIT.java rename to alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transform/tika/TikaQueueTransformServiceIT.java index 95909519..25a75feb 100644 --- a/alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transformer/TikaQueueTransformServiceIT.java +++ b/alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transform/tika/TikaQueueTransformServiceIT.java @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.tika; import static org.alfresco.transform.common.Mimetype.MIMETYPE_OPENXML_WORDPROCESSING; import static org.alfresco.transform.common.Mimetype.MIMETYPE_TEXT_PLAIN; @@ -32,6 +32,7 @@ import static org.alfresco.transform.common.Mimetype.MIMETYPE_TEXT_PLAIN; import java.util.UUID; import org.alfresco.transform.client.model.TransformRequest; +import org.alfresco.transform.base.AbstractQueueTransformServiceIT; import org.springframework.boot.test.context.SpringBootTest; /** diff --git a/alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transformer/TikaTransformationIT.java b/alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transform/tika/TikaTransformationIT.java similarity index 98% rename from alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transformer/TikaTransformationIT.java rename to alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transform/tika/TikaTransformationIT.java index 7bdf44b8..b91cc9f5 100644 --- a/alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transformer/TikaTransformationIT.java +++ b/alfresco-transform-tika/alfresco-transform-tika-boot/src/test/java/org/alfresco/transform/tika/TikaTransformationIT.java @@ -24,11 +24,11 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.tika; import static java.text.MessageFormat.format; import static java.util.function.Function.identity; -import static org.alfresco.transformer.EngineClient.sendTRequest; +import static org.alfresco.transform.base.EngineClient.sendTRequest; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.fail; import static org.springframework.http.HttpStatus.OK; diff --git a/alfresco-transform-tika/alfresco-transform-tika/pom.xml b/alfresco-transform-tika/alfresco-transform-tika/pom.xml index 6626b497..c40e621b 100644 --- a/alfresco-transform-tika/alfresco-transform-tika/pom.xml +++ b/alfresco-transform-tika/alfresco-transform-tika/pom.xml @@ -14,7 +14,7 @@ org.alfresco - alfresco-transformer-base + alfresco-t-engine-base ${project.version} diff --git a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/TikaTransformEngine.java b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/TikaTransformEngine.java new file mode 100644 index 00000000..93e1a4e1 --- /dev/null +++ b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/TikaTransformEngine.java @@ -0,0 +1,78 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.tika; + +import org.alfresco.transform.base.TransformEngine; +import org.alfresco.transform.base.probes.ProbeTestTransform; +import org.alfresco.transform.common.TransformConfigResourceReader; +import org.alfresco.transform.config.TransformConfig; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.util.Collections; + +import static org.alfresco.transform.common.Mimetype.MIMETYPE_PDF; +import static org.alfresco.transform.common.Mimetype.MIMETYPE_TEXT_PLAIN; + +@Component +public class TikaTransformEngine implements TransformEngine +{ + private static final String LICENCE = + "This transformer uses Tika from Apache. See the license at http://www.apache.org/licenses/LICENSE-2.0. or in /Apache\\ 2.0.txt\n" + + "This transformer uses ExifTool by Phil Harvey. See license at https://exiftool.org/#license. or in /Perl-Artistic-License.txt"; + + @Autowired + private TransformConfigResourceReader transformConfigResourceReader; + @Value("${transform.core.config.location:classpath:engine_config.json}") + private String engineConfigLocation; + + @Override + public String getTransformEngineName() + { + return "0001-Tika"; + } + + @Override + public String getStartupMessage() { + return LICENCE; + } + + @Override + public TransformConfig getTransformConfig() + { + return transformConfigResourceReader.read(engineConfigLocation); + } + + @Override + public ProbeTestTransform getLivenessAndReadinessProbeTestTransform() + { + return new ProbeTestTransform("quick.pdf", "quick.txt", + MIMETYPE_PDF, MIMETYPE_TEXT_PLAIN, Collections.emptyMap(), + 60, 16, 400, 10240, 60 * 30 + 1, 60 * 15 + 20); + } +} diff --git a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/metadataExtractors/AbstractTikaMetadataExtractor.java b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/metadataExtractors/AbstractTikaMetadataExtractor.java similarity index 92% rename from alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/metadataExtractors/AbstractTikaMetadataExtractor.java rename to alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/metadataExtractors/AbstractTikaMetadataExtractor.java index e52e9394..1d7f3613 100644 --- a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/metadataExtractors/AbstractTikaMetadataExtractor.java +++ b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/metadataExtractors/AbstractTikaMetadataExtractor.java @@ -24,13 +24,15 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer.metadataExtractors; +package org.alfresco.transform.tika.metadataExtractors; +import org.alfresco.transform.base.CustomTransformer; +import org.alfresco.transform.common.TransformException; +import org.alfresco.transform.base.metadataExtractors.AbstractMetadataExtractor; import org.apache.tika.embedder.Embedder; import org.apache.tika.extractor.DocumentSelector; import org.apache.tika.metadata.DublinCore; import org.apache.tika.metadata.Metadata; -import org.apache.tika.metadata.OfficeOpenXMLCore; import org.apache.tika.metadata.Property; import org.apache.tika.metadata.TikaCoreProperties; import org.apache.tika.parser.ParseContext; @@ -67,6 +69,8 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.alfresco.transform.tika.metadataExtractors.AbstractTikaMetadataExtractor.Type.EXTRACTOR; + /** * The parent of all Metadata Extractors which use Apache Tika under the hood. This handles all the * common parts of processing the files, and the common mappings. @@ -82,7 +86,7 @@ import java.util.stream.Stream; * @author Nick Burch * @author adavis */ -public abstract class AbstractTikaMetadataExtractor extends AbstractMetadataExtractor +public abstract class AbstractTikaMetadataExtractor extends AbstractMetadataExtractor implements CustomTransformer { protected static final String KEY_AUTHOR = "author"; protected static final String KEY_TITLE = "title"; @@ -97,9 +101,17 @@ public abstract class AbstractTikaMetadataExtractor extends AbstractMetadataExtr private final DateTimeFormatter tikaUTCDateFormater; private final DateTimeFormatter tikaDateFormater; - public AbstractTikaMetadataExtractor(Logger logger) + public static enum Type + { + EXTRACTOR, EMBEDDER + } + + private final Type type; + + public AbstractTikaMetadataExtractor(Type type, Logger logger) { super(logger); + this.type = type; // TODO Once TIKA-451 is fixed this list will get nicer DateTimeParser[] parsersUTC = { @@ -118,6 +130,26 @@ public abstract class AbstractTikaMetadataExtractor extends AbstractMetadataExtr tikaDateFormater = new DateTimeFormatterBuilder().append(null, parsers).toFormatter(); } + @Override + public String getTransformerName() { + return getClass().getSimpleName(); + } + + @Override + public void transform(String sourceMimetype, String sourceEncoding, InputStream inputStream, + String targetMimetype, String targetEncoding, OutputStream outputStream, + Map transformOptions) throws Exception + { + if (type == EXTRACTOR) + { + extractMetadata(sourceMimetype, transformOptions, sourceEncoding, inputStream, targetEncoding, outputStream); + } + else + { + embedMetadata(sourceMimetype, transformOptions, sourceEncoding, inputStream, targetEncoding, outputStream); + } + } + /** * Version which also tries the ISO-8601 formats (in order..), * and similar formats, which Tika makes use of @@ -308,6 +340,14 @@ public abstract class AbstractTikaMetadataExtractor extends AbstractMetadataExtr return rawProperties; } + public void embedMetadata(String sourceMimetype, Map transformOptions, + String sourceEncoding, InputStream inputStream, + String targetEncoding, OutputStream outputStream) throws Exception + { + // TODO + throw new TransformException(500, "TODO embedMetadata"); + } + /** * @deprecated The content repository's TikaPoweredMetadataExtracter provides no non test implementations. * This code exists in case there are custom implementations, that need to be converted to T-Engines. diff --git a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/metadataExtractors/DWGMetadataExtractor.java b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/metadataExtractors/DWGMetadataExtractor.java similarity index 89% rename from alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/metadataExtractors/DWGMetadataExtractor.java rename to alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/metadataExtractors/DWGMetadataExtractor.java index 47f300ed..125395ae 100644 --- a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/metadataExtractors/DWGMetadataExtractor.java +++ b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/metadataExtractors/DWGMetadataExtractor.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2021 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer.metadataExtractors; +package org.alfresco.transform.tika.metadataExtractors; import org.apache.tika.metadata.Metadata; import org.apache.tika.metadata.TikaCoreProperties; @@ -32,10 +32,13 @@ import org.apache.tika.parser.Parser; import org.apache.tika.parser.dwg.DWGParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; import java.io.Serializable; import java.util.Map; +import static org.alfresco.transform.tika.metadataExtractors.AbstractTikaMetadataExtractor.Type.EXTRACTOR; + /** * {@code "application/dwg"} and {@code "image/vnd.dwg"} metadata extractor. * @@ -53,6 +56,7 @@ import java.util.Map; * @author Nick Burch * @author adavis */ +@Component public class DWGMetadataExtractor extends AbstractTikaMetadataExtractor { private static final Logger logger = LoggerFactory.getLogger(DWGMetadataExtractor.class); @@ -62,7 +66,7 @@ public class DWGMetadataExtractor extends AbstractTikaMetadataExtractor public DWGMetadataExtractor() { - super(logger); + super(EXTRACTOR, logger); } @Override diff --git a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/metadataExtractors/IPTCMetadataExtractor.java b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/metadataExtractors/IPTCMetadataExtractor.java similarity index 91% rename from alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/metadataExtractors/IPTCMetadataExtractor.java rename to alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/metadataExtractors/IPTCMetadataExtractor.java index 91597a23..36022d0a 100644 --- a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/metadataExtractors/IPTCMetadataExtractor.java +++ b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/metadataExtractors/IPTCMetadataExtractor.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2021 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer.metadataExtractors; +package org.alfresco.transform.tika.metadataExtractors; import java.io.Serializable; import java.util.Arrays; @@ -33,13 +33,17 @@ import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.alfresco.transformer.tika.parsers.ExifToolParser; +import org.alfresco.transform.tika.parsers.ExifToolParser; import org.apache.commons.lang3.StringUtils; import org.apache.tika.metadata.Metadata; import org.apache.tika.parser.Parser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; +import static org.alfresco.transform.tika.metadataExtractors.AbstractTikaMetadataExtractor.Type.EXTRACTOR; + +@Component public class IPTCMetadataExtractor extends AbstractTikaMetadataExtractor { @@ -53,7 +57,7 @@ public class IPTCMetadataExtractor extends AbstractTikaMetadataExtractor public IPTCMetadataExtractor() { - super(logger); + super(EXTRACTOR, logger); } @Override diff --git a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/metadataExtractors/MP3MetadataExtractor.java b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/metadataExtractors/MP3MetadataExtractor.java similarity index 93% rename from alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/metadataExtractors/MP3MetadataExtractor.java rename to alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/metadataExtractors/MP3MetadataExtractor.java index b6cd2826..7467767b 100644 --- a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/metadataExtractors/MP3MetadataExtractor.java +++ b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/metadataExtractors/MP3MetadataExtractor.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2021 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer.metadataExtractors; +package org.alfresco.transform.tika.metadataExtractors; import org.apache.tika.metadata.Metadata; import org.apache.tika.metadata.TikaCoreProperties; @@ -33,10 +33,13 @@ import org.apache.tika.parser.Parser; import org.apache.tika.parser.mp3.Mp3Parser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; import java.io.Serializable; import java.util.Map; +import static org.alfresco.transform.tika.metadataExtractors.AbstractTikaMetadataExtractor.Type.EXTRACTOR; + /** * MP3 file metadata extractor. * @@ -63,6 +66,7 @@ import java.util.Map; * @author Nick Burch * @author adavis */ +@Component public class MP3MetadataExtractor extends TikaAudioMetadataExtractor { private static final Logger logger = LoggerFactory.getLogger(MP3MetadataExtractor.class); diff --git a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/metadataExtractors/MailMetadataExtractor.java b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/metadataExtractors/MailMetadataExtractor.java similarity index 93% rename from alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/metadataExtractors/MailMetadataExtractor.java rename to alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/metadataExtractors/MailMetadataExtractor.java index 86d168c6..82379634 100644 --- a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/metadataExtractors/MailMetadataExtractor.java +++ b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/metadataExtractors/MailMetadataExtractor.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2021 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer.metadataExtractors; +package org.alfresco.transform.tika.metadataExtractors; import org.apache.tika.metadata.Message; import org.apache.tika.metadata.Metadata; @@ -33,10 +33,13 @@ import org.apache.tika.parser.Parser; import org.apache.tika.parser.microsoft.OfficeParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; import java.io.Serializable; import java.util.Map; +import static org.alfresco.transform.tika.metadataExtractors.AbstractTikaMetadataExtractor.Type.EXTRACTOR; + /** * Outlook MAPI format email metadata extractor. * @@ -59,6 +62,7 @@ import java.util.Map; * @author Kevin Roast * @author adavis */ +@Component public class MailMetadataExtractor extends AbstractTikaMetadataExtractor { private static final Logger logger = LoggerFactory.getLogger(MailMetadataExtractor.class); @@ -74,7 +78,7 @@ public class MailMetadataExtractor extends AbstractTikaMetadataExtractor public MailMetadataExtractor() { - super(logger); + super(EXTRACTOR, logger); } @Override diff --git a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/metadataExtractors/OfficeMetadataExtractor.java b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/metadataExtractors/OfficeMetadataExtractor.java similarity index 93% rename from alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/metadataExtractors/OfficeMetadataExtractor.java rename to alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/metadataExtractors/OfficeMetadataExtractor.java index 7612a386..eed591cf 100644 --- a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/metadataExtractors/OfficeMetadataExtractor.java +++ b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/metadataExtractors/OfficeMetadataExtractor.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2021 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer.metadataExtractors; +package org.alfresco.transform.tika.metadataExtractors; import org.apache.tika.metadata.Metadata; import org.apache.tika.metadata.Office; @@ -33,10 +33,13 @@ import org.apache.tika.parser.Parser; import org.apache.tika.parser.microsoft.OfficeParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; import java.io.Serializable; import java.util.Map; +import static org.alfresco.transform.tika.metadataExtractors.AbstractTikaMetadataExtractor.Type.EXTRACTOR; + /** * Office file format metadata extractor. * @@ -67,6 +70,7 @@ import java.util.Map; * @author Nick Burch * @author adavis */ +@Component public class OfficeMetadataExtractor extends AbstractTikaMetadataExtractor { private static final Logger logger = LoggerFactory.getLogger(OfficeMetadataExtractor.class); @@ -84,7 +88,7 @@ public class OfficeMetadataExtractor extends AbstractTikaMetadataExtractor public OfficeMetadataExtractor() { - super(logger); + super(EXTRACTOR, logger); } @Override diff --git a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/metadataExtractors/OpenDocumentMetadataExtractor.java b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/metadataExtractors/OpenDocumentMetadataExtractor.java similarity index 95% rename from alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/metadataExtractors/OpenDocumentMetadataExtractor.java rename to alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/metadataExtractors/OpenDocumentMetadataExtractor.java index 8014802b..dd53e207 100644 --- a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/metadataExtractors/OpenDocumentMetadataExtractor.java +++ b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/metadataExtractors/OpenDocumentMetadataExtractor.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2021 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,8 +24,9 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer.metadataExtractors; +package org.alfresco.transform.tika.metadataExtractors; +import static org.alfresco.transform.tika.metadataExtractors.AbstractTikaMetadataExtractor.Type.EXTRACTOR; import static org.apache.tika.metadata.DublinCore.NAMESPACE_URI_DC; import org.apache.tika.metadata.Metadata; @@ -40,6 +41,7 @@ import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; import org.xml.sax.ContentHandler; import java.io.Serializable; @@ -76,6 +78,7 @@ import java.util.stream.Collectors; * @author Derek Hulley * @author adavis */ +@Component public class OpenDocumentMetadataExtractor extends AbstractTikaMetadataExtractor { private static final Logger logger = LoggerFactory.getLogger(OpenDocumentMetadataExtractor.class); @@ -95,7 +98,7 @@ public class OpenDocumentMetadataExtractor extends AbstractTikaMetadataExtractor public OpenDocumentMetadataExtractor() { - super(logger); + super(EXTRACTOR, logger); } @Override diff --git a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/metadataExtractors/PdfBoxMetadataExtractor.java b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/metadataExtractors/PdfBoxMetadataExtractor.java similarity index 85% rename from alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/metadataExtractors/PdfBoxMetadataExtractor.java rename to alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/metadataExtractors/PdfBoxMetadataExtractor.java index 83acab10..9c7dcec6 100644 --- a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/metadataExtractors/PdfBoxMetadataExtractor.java +++ b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/metadataExtractors/PdfBoxMetadataExtractor.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2020 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,15 +24,18 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer.metadataExtractors; +package org.alfresco.transform.tika.metadataExtractors; -import org.alfresco.transformer.executors.Tika; +import org.alfresco.transform.tika.transformers.Tika; import org.apache.tika.extractor.DocumentSelector; import org.apache.tika.metadata.Metadata; import org.apache.tika.parser.Parser; import org.apache.tika.parser.pdf.PDFParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import static org.alfresco.transform.tika.metadataExtractors.AbstractTikaMetadataExtractor.Type.EXTRACTOR; /** * Metadata extractor for the PDF documents. @@ -52,13 +55,14 @@ import org.slf4j.LoggerFactory; * @author Derek Hulley * @author adavis */ +@Component public class PdfBoxMetadataExtractor extends AbstractTikaMetadataExtractor { private static final Logger logger = LoggerFactory.getLogger(PdfBoxMetadataExtractor.class); public PdfBoxMetadataExtractor() { - super(logger); + super(EXTRACTOR, logger); } @Override diff --git a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/metadataExtractors/PoiMetadataExtractor.java b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/metadataExtractors/PoiMetadataExtractor.java similarity index 94% rename from alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/metadataExtractors/PoiMetadataExtractor.java rename to alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/metadataExtractors/PoiMetadataExtractor.java index 71cdd7a8..f906919c 100644 --- a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/metadataExtractors/PoiMetadataExtractor.java +++ b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/metadataExtractors/PoiMetadataExtractor.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2021 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer.metadataExtractors; +package org.alfresco.transform.tika.metadataExtractors; import org.apache.poi.ooxml.POIXMLProperties; import org.apache.poi.xssf.usermodel.XSSFWorkbook; @@ -36,6 +36,7 @@ import org.apache.tika.parser.Parser; import org.apache.tika.parser.microsoft.ooxml.OOXMLParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; import java.io.IOException; import java.io.InputStream; @@ -44,6 +45,8 @@ import java.util.Collections; import java.util.Set; import java.util.StringJoiner; +import static org.alfresco.transform.tika.metadataExtractors.AbstractTikaMetadataExtractor.Type.EXTRACTOR; + /** * POI-based metadata extractor for Office 07 documents. See http://poi.apache.org/ for information on POI. * @@ -92,13 +95,14 @@ import java.util.StringJoiner; * @author Dmitry Velichkevich * @author adavis */ +@Component public class PoiMetadataExtractor extends AbstractTikaMetadataExtractor { private static final Logger logger = LoggerFactory.getLogger(PoiMetadataExtractor.class); public PoiMetadataExtractor() { - super(logger); + super(EXTRACTOR, logger); } @Override diff --git a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/metadataExtractors/TikaAudioMetadataExtractor.java b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/metadataExtractors/TikaAudioMetadataExtractor.java similarity index 93% rename from alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/metadataExtractors/TikaAudioMetadataExtractor.java rename to alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/metadataExtractors/TikaAudioMetadataExtractor.java index e7933ef3..d0e9e5f5 100644 --- a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/metadataExtractors/TikaAudioMetadataExtractor.java +++ b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/metadataExtractors/TikaAudioMetadataExtractor.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2021 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer.metadataExtractors; +package org.alfresco.transform.tika.metadataExtractors; import org.apache.tika.config.TikaConfig; import org.apache.tika.metadata.Metadata; @@ -37,12 +37,14 @@ import org.gagravarr.tika.FlacParser; import org.gagravarr.tika.VorbisParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; import java.io.Serializable; import java.util.Calendar; import java.util.Map; -import static org.alfresco.transformer.executors.Tika.readTikaConfig; +import static org.alfresco.transform.tika.transformers.Tika.readTikaConfig; +import static org.alfresco.transform.tika.metadataExtractors.AbstractTikaMetadataExtractor.Type.EXTRACTOR; /** * A Metadata Extractor which makes use of the Apache Tika Audio Parsers to extract metadata from media files. @@ -66,6 +68,7 @@ import static org.alfresco.transformer.executors.Tika.readTikaConfig; * @author Nick Burch * @author adavis */ +@Component public class TikaAudioMetadataExtractor extends AbstractTikaMetadataExtractor { private static final Logger logger = LoggerFactory.getLogger(TikaAudioMetadataExtractor.class); @@ -86,7 +89,7 @@ public class TikaAudioMetadataExtractor extends AbstractTikaMetadataExtractor public TikaAudioMetadataExtractor(Logger logger) { - super(logger); + super(EXTRACTOR, logger); tikaConfig = readTikaConfig(logger); } diff --git a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/metadataExtractors/TikaAutoMetadataExtractor.java b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/metadataExtractors/TikaAutoMetadataExtractor.java similarity index 94% rename from alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/metadataExtractors/TikaAutoMetadataExtractor.java rename to alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/metadataExtractors/TikaAutoMetadataExtractor.java index 0f6a6dd7..0d0edd7c 100644 --- a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/metadataExtractors/TikaAutoMetadataExtractor.java +++ b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/metadataExtractors/TikaAutoMetadataExtractor.java @@ -24,9 +24,8 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer.metadataExtractors; +package org.alfresco.transform.tika.metadataExtractors; -import org.alfresco.transform.common.Mimetype; import org.apache.tika.config.TikaConfig; import org.apache.tika.metadata.Metadata; import org.apache.tika.metadata.TIFF; @@ -34,12 +33,14 @@ import org.apache.tika.parser.AutoDetectParser; import org.apache.tika.parser.Parser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; import java.io.Serializable; import java.util.Map; import static org.alfresco.transform.common.Mimetype.MIMETYPE_IMAGE_JPEG; -import static org.alfresco.transformer.executors.Tika.readTikaConfig; +import static org.alfresco.transform.tika.transformers.Tika.readTikaConfig; +import static org.alfresco.transform.tika.metadataExtractors.AbstractTikaMetadataExtractor.Type.EXTRACTOR; /** * A Metadata Extractor which makes use of the Apache Tika auto-detection to select the best parser to extract the @@ -61,6 +62,7 @@ import static org.alfresco.transformer.executors.Tika.readTikaConfig; * @author Nick Burch * @author adavis */ +@Component public class TikaAutoMetadataExtractor extends AbstractTikaMetadataExtractor { private static final Logger logger = LoggerFactory.getLogger(TikaAutoMetadataExtractor.class); @@ -75,7 +77,7 @@ public class TikaAutoMetadataExtractor extends AbstractTikaMetadataExtractor public TikaAutoMetadataExtractor() { - super(logger); + super(EXTRACTOR, logger); tikaConfig = readTikaConfig(logger); } diff --git a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/tika/parsers/ExifToolParser.java b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/parsers/ExifToolParser.java similarity index 97% rename from alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/tika/parsers/ExifToolParser.java rename to alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/parsers/ExifToolParser.java index d10230a3..660fd82a 100644 --- a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/tika/parsers/ExifToolParser.java +++ b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/parsers/ExifToolParser.java @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer.tika.parsers; +package org.alfresco.transform.tika.parsers; import static java.nio.charset.StandardCharsets.UTF_8; import static org.alfresco.transform.common.Mimetype.MIMETYPE_IMAGE_JPEG; diff --git a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/executors/TikaOfficeDetectParser.java b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/parsers/TikaOfficeDetectParser.java similarity index 97% rename from alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/executors/TikaOfficeDetectParser.java rename to alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/parsers/TikaOfficeDetectParser.java index 6600e472..2f188a12 100644 --- a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/executors/TikaOfficeDetectParser.java +++ b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/parsers/TikaOfficeDetectParser.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2019 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer.executors; +package org.alfresco.transform.tika.parsers; import java.io.IOException; import java.io.InputStream; diff --git a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/ArchiveTransformer.java b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/ArchiveTransformer.java new file mode 100644 index 00000000..68bbf943 --- /dev/null +++ b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/ArchiveTransformer.java @@ -0,0 +1,42 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.tika.transformers; + +import org.apache.tika.parser.Parser; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; + +@Component +public class ArchiveTransformer extends GenericTikaTransformer +{ + @Override + protected Parser getParser() + { + return tika.packageParser; + } +} diff --git a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/GenericTikaTransformer.java b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/GenericTikaTransformer.java new file mode 100644 index 00000000..65f10506 --- /dev/null +++ b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/GenericTikaTransformer.java @@ -0,0 +1,146 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.tika.transformers; + +import org.alfresco.transform.base.CustomTransformer; +import org.alfresco.transform.base.logging.LogEntry; +import org.alfresco.transform.base.util.RequestParamMap; +import org.alfresco.transform.common.TransformException; +import org.apache.tika.extractor.DocumentSelector; +import org.apache.tika.parser.Parser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; + +import java.io.File; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Map; +import java.util.StringJoiner; + +import static java.lang.Boolean.parseBoolean; + +public abstract class GenericTikaTransformer implements CustomTransformer +{ + private static final Logger logger = LoggerFactory.getLogger(GenericTikaTransformer.class); + + @Value("${transform.core.tika.pdfBox.notExtractBookmarksTextDefault:false}") + boolean notExtractBookmarksTextDefault; + + @Autowired + protected Tika tika; + + protected abstract Parser getParser(); + + protected DocumentSelector getDocumentSelector() + { + return null; + } + + @Override + public String getTransformerName() + { + String simpleClassName = getClass().getSimpleName(); + return simpleClassName.substring(0, simpleClassName.length()-"Transformer".length()); + } + + @Override + public void transform(String sourceMimetype, String sourceEncoding, InputStream inputStream, + String targetMimetype, String targetEncoding, OutputStream outputStream, + Map transformOptions) throws Exception + { + // TODO + throw new TransformException(500, "TODO GenericTikaTransformer transform with InputStreams"); + } + + public void transform(String transformName, String sourceMimetype, String targetMimetype, + Map transformOptions, File sourceFile, File targetFile) + throws Exception + { + final boolean includeContents = parseBoolean( + transformOptions.getOrDefault(RequestParamMap.INCLUDE_CONTENTS, "false")); + final boolean notExtractBookmarksText = parseBoolean( + transformOptions.getOrDefault(RequestParamMap.NOT_EXTRACT_BOOKMARKS_TEXT, String.valueOf(notExtractBookmarksTextDefault))); + final String targetEncoding = transformOptions.getOrDefault("targetEncoding", "UTF-8"); + if (transformOptions.get(RequestParamMap.NOT_EXTRACT_BOOKMARKS_TEXT) == null && notExtractBookmarksTextDefault) + { + logger.trace("notExtractBookmarksText default value has been overridden to {}", notExtractBookmarksTextDefault); + } + call(sourceFile, targetFile, transformName, + includeContents ? Tika.INCLUDE_CONTENTS : null, + notExtractBookmarksText ? Tika.NOT_EXTRACT_BOOKMARKS_TEXT : null, + Tika.TARGET_MIMETYPE + targetMimetype, Tika.TARGET_ENCODING + targetEncoding); + } + + void call(File sourceFile, File targetFile, String... args) + { + Parser parser = getParser(); + DocumentSelector documentSelector = getDocumentSelector(); + args = buildArgs(sourceFile, targetFile, args); + tika.transform(parser, documentSelector, args); + } + + private static String[] buildArgs(File sourceFile, File targetFile, String[] args) + { + ArrayList methodArgs = new ArrayList<>(args.length + 2); + StringJoiner sj = new StringJoiner(" "); + for (String arg : args) + { + addArg(methodArgs, sj, arg); + } + + addFileArg(methodArgs, sj, sourceFile); + addFileArg(methodArgs, sj, targetFile); + + LogEntry.setOptions(sj.toString()); + + return methodArgs.toArray(new String[0]); + } + + private static void addArg(ArrayList methodArgs, StringJoiner sj, String arg) + { + if (arg != null) + { + sj.add(arg); + methodArgs.add(arg); + } + } + + private static void addFileArg(ArrayList methodArgs, StringJoiner sj, File arg) + { + if (arg != null) + { + String path = arg.getAbsolutePath(); + int i = path.lastIndexOf('.'); + String ext = i == -1 ? "???" : path.substring(i + 1); + sj.add(ext); + methodArgs.add(path); + } + } +} diff --git a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/OOXMLTransformer.java b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/OOXMLTransformer.java new file mode 100644 index 00000000..ffafdbd6 --- /dev/null +++ b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/OOXMLTransformer.java @@ -0,0 +1,40 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2021 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.tika.transformers; + +import org.apache.tika.parser.Parser; +import org.springframework.stereotype.Component; + +@Component +public class OOXMLTransformer extends GenericTikaTransformer +{ + @Override + protected Parser getParser() + { + return tika.ooXmlParser; + } +} diff --git a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/OfficeTransformer.java b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/OfficeTransformer.java new file mode 100644 index 00000000..a9abdeb3 --- /dev/null +++ b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/OfficeTransformer.java @@ -0,0 +1,40 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2021 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.tika.transformers; + +import org.apache.tika.parser.Parser; +import org.springframework.stereotype.Component; + +@Component +public class OfficeTransformer extends GenericTikaTransformer +{ + @Override + protected Parser getParser() + { + return tika.officeParser; + } +} diff --git a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/OutlookMsgTransformer.java b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/OutlookMsgTransformer.java new file mode 100644 index 00000000..f4ffa72d --- /dev/null +++ b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/OutlookMsgTransformer.java @@ -0,0 +1,40 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2021 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.tika.transformers; + +import org.apache.tika.parser.Parser; +import org.springframework.stereotype.Component; + +@Component +public class OutlookMsgTransformer extends GenericTikaTransformer +{ + @Override + protected Parser getParser() + { + return tika.officeParser; + } +} diff --git a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/PdfBoxTransformer.java b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/PdfBoxTransformer.java new file mode 100644 index 00000000..11c686da --- /dev/null +++ b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/PdfBoxTransformer.java @@ -0,0 +1,47 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2021 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.tika.transformers; + +import org.apache.tika.extractor.DocumentSelector; +import org.apache.tika.parser.Parser; +import org.springframework.stereotype.Component; + +@Component +public class PdfBoxTransformer extends GenericTikaTransformer +{ + @Override + protected Parser getParser() + { + return tika.pdfParser; + } + + @Override + protected DocumentSelector getDocumentSelector() + { + return tika.pdfBoxEmbededDocumentSelector; + } +} diff --git a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/PoiTransformer.java b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/PoiTransformer.java new file mode 100644 index 00000000..d26f98e4 --- /dev/null +++ b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/PoiTransformer.java @@ -0,0 +1,40 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2021 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.tika.transformers; + +import org.apache.tika.parser.Parser; +import org.springframework.stereotype.Component; + +@Component +public class PoiTransformer extends GenericTikaTransformer +{ + @Override + protected Parser getParser() + { + return tika.tikaOfficeDetectParser; + } +} diff --git a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/TextMiningTransformer.java b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/TextMiningTransformer.java new file mode 100644 index 00000000..7b611887 --- /dev/null +++ b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/TextMiningTransformer.java @@ -0,0 +1,40 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2021 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.tika.transformers; + +import org.apache.tika.parser.Parser; +import org.springframework.stereotype.Component; + +@Component +public class TextMiningTransformer extends GenericTikaTransformer +{ + @Override + protected Parser getParser() + { + return tika.officeParser; + } +} diff --git a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/Tika.java b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/Tika.java new file mode 100644 index 00000000..e3064683 --- /dev/null +++ b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/Tika.java @@ -0,0 +1,446 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.tika.transformers; + +import com.google.common.collect.ImmutableList; +import org.alfresco.transform.tika.parsers.TikaOfficeDetectParser; +import org.apache.tika.config.TikaConfig; +import org.apache.tika.exception.TikaException; +import org.apache.tika.extractor.DocumentSelector; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.AutoDetectParser; +import org.apache.tika.parser.EmptyParser; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.parser.Parser; +import org.apache.tika.parser.microsoft.OfficeParser; +import org.apache.tika.parser.microsoft.ooxml.OOXMLParser; +import org.apache.tika.parser.pdf.PDFParser; +import org.apache.tika.parser.pdf.PDFParserConfig; +import org.apache.tika.parser.pkg.PackageParser; +import org.apache.tika.sax.BodyContentHandler; +import org.apache.tika.sax.ExpandedTitleContentHandler; +import org.slf4j.Logger; +import org.springframework.stereotype.Component; +import org.xml.sax.Attributes; +import org.xml.sax.ContentHandler; +import org.xml.sax.SAXException; + +import javax.xml.transform.OutputKeys; +import javax.xml.transform.TransformerConfigurationException; +import javax.xml.transform.sax.SAXTransformerFactory; +import javax.xml.transform.sax.TransformerHandler; +import javax.xml.transform.stream.StreamResult; +import java.io.BufferedInputStream; +import java.io.BufferedWriter; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.net.URL; +import java.util.List; +import java.util.regex.Pattern; + +import static org.alfresco.transform.common.Mimetype.MIMETYPE_HTML; +import static org.alfresco.transform.common.Mimetype.MIMETYPE_IMAGE_JPEG; +import static org.alfresco.transform.common.Mimetype.MIMETYPE_IMAGE_PNG; +import static org.alfresco.transform.common.Mimetype.MIMETYPE_IMAGE_TIFF; +import static org.alfresco.transform.common.Mimetype.MIMETYPE_TEXT_CSV; +import static org.alfresco.transform.common.Mimetype.MIMETYPE_TEXT_PLAIN; +import static org.alfresco.transform.common.Mimetype.MIMETYPE_XHTML; +import static org.alfresco.transform.common.Mimetype.MIMETYPE_XML; + +@Component +public class Tika +{ + public static final String ARCHIVE = "Archive"; + public static final String OUTLOOK_MSG = "OutlookMsg"; + public static final String PDF_BOX = "PdfBox"; + public static final String OFFICE = "Office"; + public static final String POI = "Poi"; + public static final String OOXML = "OOXML"; + public static final String TIKA_AUTO = "TikaAuto"; + public static final String TEXT_MINING = "TextMining"; + + public static final String TARGET_MIMETYPE = "--targetMimetype="; + public static final String TARGET_ENCODING = "--targetEncoding="; + public static final String INCLUDE_CONTENTS = "--includeContents"; + public static final String NOT_EXTRACT_BOOKMARKS_TEXT = "--notExtractBookmarksText"; + + public static final String CSV = "csv"; + public static final String DOC = "doc"; + public static final String DOCX = "docx"; + public static final String HTML = "html"; + public static final String MSG = "msg"; + public static final String PDF = "pdf"; + public static final String PPTX = "pptx"; + public static final String TXT = "txt"; + public static final String XHTML = "xhtml"; + public static final String XSLX = "xslx"; + public static final String XML = "xml"; + public static final String ZIP = "zip"; + + public static final Parser packageParser = new PackageParser(); + public static final Parser pdfParser = new PDFParser(); + public static final Parser officeParser = new OfficeParser(); + public final Parser autoDetectParser; + public static final Parser ooXmlParser = new OOXMLParser(); + public static final Parser tikaOfficeDetectParser = new TikaOfficeDetectParser(); + public final PDFParserConfig pdfParserConfig = new PDFParserConfig(); + + public static final DocumentSelector pdfBoxEmbededDocumentSelector = new DocumentSelector() + { + private final List disabledMediaTypes = ImmutableList.of(MIMETYPE_IMAGE_JPEG, + MIMETYPE_IMAGE_TIFF, MIMETYPE_IMAGE_PNG); + + @Override + public boolean select(Metadata metadata) + { + String contentType = metadata.get(Metadata.CONTENT_TYPE); + if (contentType == null || contentType.equals("") || disabledMediaTypes == null) + { + return true; + } + return !disabledMediaTypes.contains(contentType); + } + }; + + public Tika() throws TikaException, IOException, SAXException + { + TikaConfig tikaConfig = readTikaConfig(); + autoDetectParser = new AutoDetectParser(tikaConfig); + } + + public static TikaConfig readTikaConfig(Logger logger) + { + try + { + return readTikaConfig(); + } + catch (Exception e) + { + logger.error("Failed to read tika-config.xml", e); + return null; + } + } + + private static TikaConfig readTikaConfig() throws TikaException, IOException, SAXException + { + ClassLoader classLoader = Tika.class.getClassLoader(); + URL tikaConfigXml = classLoader.getResource("tika-config.xml"); + return new TikaConfig(tikaConfigXml); + } + + // Extracts parameters form args + public void transform(Parser parser, DocumentSelector documentSelector, String[] args) + { + String transform = null; + String targetMimetype = null; + String targetEncoding = null; + String sourceFilename = null; + String targetFilename = null; + Boolean includeContents = null; + Boolean notExtractBookmarksText = null; + + for (String arg : args) + { + if (arg.startsWith("--")) + { + if (INCLUDE_CONTENTS.startsWith(arg)) + { + getValue(arg, false, includeContents, INCLUDE_CONTENTS); + includeContents = true; + } + else if (arg.startsWith(TARGET_ENCODING)) + { + targetEncoding = getValue(arg, true, targetEncoding, TARGET_ENCODING); + } + else if (arg.startsWith(TARGET_MIMETYPE)) + { + targetMimetype = getValue(arg, true, targetMimetype, TARGET_MIMETYPE); + } + else if (arg.startsWith(NOT_EXTRACT_BOOKMARKS_TEXT)) + { + getValue(arg, false, notExtractBookmarksText, NOT_EXTRACT_BOOKMARKS_TEXT); + notExtractBookmarksText = true; + } + else + { + throw new IllegalArgumentException("Unexpected argument " + arg); + } + } + else + { + if (transform == null) + { + transform = arg; + } + else if (sourceFilename == null) + { + sourceFilename = arg; + } + else if (targetFilename == null) + { + targetFilename = arg; + } + else + { + throw new IllegalArgumentException("Unexpected argument " + arg); + } + } + } + if (targetFilename == null) + { + throw new IllegalArgumentException("Missing arguments"); + } + includeContents = includeContents == null ? false : includeContents; + notExtractBookmarksText = notExtractBookmarksText == null ? false : notExtractBookmarksText; + + transform(parser, documentSelector, includeContents, notExtractBookmarksText, sourceFilename, + targetFilename, targetMimetype, targetEncoding); + } + + private String getValue(String arg, boolean valueExpected, Object value, String optionName) + { + if (value != null) + { + throw new IllegalArgumentException("Duplicate " + optionName); + } + String stringValue = arg.substring(optionName.length()).trim(); + if (!valueExpected && stringValue.length() > 0) + { + throw new IllegalArgumentException("Unexpected value with " + optionName); + } + if (valueExpected && stringValue.length() == 0) + { + throw new IllegalArgumentException("Expected value with " + optionName); + } + return stringValue; + } + + private void transform(Parser parser, DocumentSelector documentSelector, + Boolean includeContents, + Boolean notExtractBookmarksText, + String sourceFilename, + String targetFilename, String targetMimetype, String targetEncoding) + { + + try (InputStream is = new BufferedInputStream(new FileInputStream(sourceFilename)); + OutputStream os = new FileOutputStream(targetFilename); + Writer ow = new BufferedWriter(new OutputStreamWriter(os, targetEncoding))) + { + Metadata metadata = new Metadata(); + ParseContext context = buildParseContext(documentSelector, includeContents, + notExtractBookmarksText); + ContentHandler handler = getContentHandler(targetMimetype, ow); + + parser.parse(is, handler, metadata, context); + } + catch (SAXException | TikaException | IOException e) + { + throw new IllegalStateException(e.getMessage(), e); + } + } + + private ContentHandler getContentHandler(String targetMimetype, Writer output) + { + try + { + ContentHandler handler; + if (MIMETYPE_TEXT_PLAIN.equals(targetMimetype)) + { + handler = new BodyContentHandler(output); + } + else + { + SAXTransformerFactory factory = (SAXTransformerFactory) SAXTransformerFactory.newInstance(); + TransformerHandler transformerHandler; + transformerHandler = factory.newTransformerHandler(); + transformerHandler.getTransformer().setOutputProperty(OutputKeys.INDENT, "yes"); + transformerHandler.setResult(new StreamResult(output)); + handler = transformerHandler; + + if (MIMETYPE_HTML.equals(targetMimetype)) + { + transformerHandler.getTransformer().setOutputProperty(OutputKeys.METHOD, HTML); + return new ExpandedTitleContentHandler(transformerHandler); + } + else if (MIMETYPE_XHTML.equals(targetMimetype) || + MIMETYPE_XML.equals(targetMimetype)) + { + transformerHandler.getTransformer().setOutputProperty(OutputKeys.METHOD, XML); + } + else if (MIMETYPE_TEXT_CSV.equals(targetMimetype)) + { + handler = new CsvContentHandler(output); + } + else + { + throw new IllegalArgumentException("Invalid target mimetype " + targetMimetype); + } + } + return handler; + } + catch (TransformerConfigurationException e) + { + throw new IllegalStateException(e.getMessage(), e); + } + } + + /** + * A wrapper around the normal Tika BodyContentHandler for CSV rather encoding than tab separated. + */ + protected static class CsvContentHandler extends BodyContentHandler + { + private static final char[] comma = new char[]{','}; + private static final Pattern all_nums = Pattern.compile("[\\d\\.\\-\\+]+"); + + private boolean inCell = false; + private boolean needsComma = false; + + protected CsvContentHandler(Writer output) + { + super(output); + } + + @Override + public void ignorableWhitespace(char[] ch, int start, int length) + throws SAXException + { + if (length == 1 && ch[0] == '\t') + { + // Ignore tabs, as they mess up the CSV output + } + else + { + super.ignorableWhitespace(ch, start, length); + } + } + + @Override + public void characters(char[] ch, int start, int length) + throws SAXException + { + if (inCell) + { + StringBuffer t = new StringBuffer(new String(ch, start, length)); + + // Quote if not all numbers + if (all_nums.matcher(t).matches()) + { + super.characters(ch, start, length); + } + else + { + for (int i = t.length() - 1; i >= 0; i--) + { + if (t.charAt(i) == '\"') + { + // Double up double quotes + t.insert(i, '\"'); + i--; + } + } + t.insert(0, '\"'); + t.append('\"'); + char[] c = t.toString().toCharArray(); + super.characters(c, 0, c.length); + } + } + else + { + super.characters(ch, start, length); + } + } + + @Override + public void startElement(String uri, String localName, String name, + Attributes atts) throws SAXException + { + if (localName.equals("td")) + { + inCell = true; + if (needsComma) + { + super.characters(comma, 0, 1); + needsComma = true; + } + } + else + { + super.startElement(uri, localName, name, atts); + } + } + + @Override + public void endElement(String uri, String localName, String name) + throws SAXException + { + if (localName.equals("td")) + { + needsComma = true; + inCell = false; + } + else + { + if (localName.equals("tr")) + { + needsComma = false; + } + super.endElement(uri, localName, name); + } + } + } + + private ParseContext buildParseContext(DocumentSelector documentSelector, + Boolean includeContents, Boolean notExtractBookmarksText) + { + ParseContext context = new ParseContext(); + + if (documentSelector != null) + { + context.set(DocumentSelector.class, documentSelector); + } + + if (notExtractBookmarksText.equals(true)) + { + pdfParserConfig.setExtractBookmarksText(false); + // pdfParserConfig is set to override default settings + context.set(PDFParserConfig.class, pdfParserConfig); + } + + // If Archive transform + if (includeContents != null) + { + context.set(Parser.class, includeContents ? autoDetectParser : new EmptyParser()); + } + + return context; + } +} diff --git a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/TikaAutoTransformer.java b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/TikaAutoTransformer.java new file mode 100644 index 00000000..93135a5e --- /dev/null +++ b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transform/tika/transformers/TikaAutoTransformer.java @@ -0,0 +1,40 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2021 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.tika.transformers; + +import org.apache.tika.parser.Parser; +import org.springframework.stereotype.Component; + +@Component +public class TikaAutoTransformer extends GenericTikaTransformer +{ + @Override + protected Parser getParser() + { + return tika.autoDetectParser; + } +} diff --git a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/executors/Tika.java b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/executors/Tika.java deleted file mode 100644 index b1205f63..00000000 --- a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/executors/Tika.java +++ /dev/null @@ -1,876 +0,0 @@ -/* - * #%L - * Alfresco Transform Core - * %% - * Copyright (C) 2005 - 2022 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * - - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ -package org.alfresco.transformer.executors; - -import com.google.common.collect.ImmutableList; -import org.apache.tika.config.TikaConfig; -import org.apache.tika.exception.TikaException; -import org.apache.tika.extractor.DocumentSelector; -import org.apache.tika.metadata.Metadata; -import org.apache.tika.parser.AutoDetectParser; -import org.apache.tika.parser.EmptyParser; -import org.apache.tika.parser.ParseContext; -import org.apache.tika.parser.Parser; -import org.apache.tika.parser.microsoft.OfficeParser; -import org.apache.tika.parser.microsoft.ooxml.OOXMLParser; -import org.apache.tika.parser.pdf.PDFParser; -import org.apache.tika.parser.pdf.PDFParserConfig; -import org.apache.tika.parser.pkg.PackageParser; -import org.apache.tika.sax.BodyContentHandler; -import org.apache.tika.sax.ExpandedTitleContentHandler; -import org.slf4j.Logger; -import org.xml.sax.Attributes; -import org.xml.sax.ContentHandler; -import org.xml.sax.SAXException; - -import javax.xml.transform.OutputKeys; -import javax.xml.transform.TransformerConfigurationException; -import javax.xml.transform.sax.SAXTransformerFactory; -import javax.xml.transform.sax.TransformerHandler; -import javax.xml.transform.stream.StreamResult; -import java.io.BufferedInputStream; -import java.io.BufferedWriter; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.io.Writer; -import java.net.URL; -import java.util.List; -import java.util.regex.Pattern; - -import static org.alfresco.transform.common.Mimetype.MIMETYPE_HTML; -import static org.alfresco.transform.common.Mimetype.MIMETYPE_IMAGE_JPEG; -import static org.alfresco.transform.common.Mimetype.MIMETYPE_IMAGE_PNG; -import static org.alfresco.transform.common.Mimetype.MIMETYPE_IMAGE_TIFF; -import static org.alfresco.transform.common.Mimetype.MIMETYPE_TEXT_CSV; -import static org.alfresco.transform.common.Mimetype.MIMETYPE_TEXT_PLAIN; -import static org.alfresco.transform.common.Mimetype.MIMETYPE_XHTML; -import static org.alfresco.transform.common.Mimetype.MIMETYPE_XML; - -/** - * Stripped down command line Tika transformers. Not actually run as a separate process, but the code fits the patten - * used by transformers that do. - * 
- *
- *           Archive 0 ms
- *               1) cpio html [100] unlimited
- *               2) cpio txt   [50] unlimited
- *               3) cpio xhtml [100] unlimited
- *               4) cpio xml  [100] unlimited
- *               5) jar  html [100] unlimited
- *               6) jar  txt   [50] unlimited
- *               7) jar  xhtml [100] unlimited
- *               8) jar  xml  [100] unlimited
- *               9) tar  html [100] unlimited
- *              10) tar  txt   [50] unlimited
- *              11) tar  xhtml [100] unlimited
- *              12) tar  xml  [100] unlimited
- *              13) zip  html [100] unlimited
- *              14) zip  txt   [50] unlimited
- *              15) zip  xhtml [100] unlimited
- *              16) zip  xml  [100] unlimited
- *           PdfBox 0 ms
- *               1) pdf  html [110] unlimited
- *               2) pdf  txt   [50] 25 MB
- *               3) pdf  xhtml [110] unlimited
- *               4) pdf  xml  [110] unlimited
- *           OutlookMsg 0 ms
- *               1) msg  html [125] unlimited
- *               2) msg  txt  [125] unlimited
- *               3) msg  xhtml [125] unlimited
- *               4) msg  xml  [125] unlimited
- *           PdfBox 0 ms
- *               1) pdf  html [110] unlimited
- *               2) pdf  txt   [50] 25 MB
- *               3) pdf  xhtml [110] unlimited
- *               4) pdf  xml  [110] unlimited
- *           Office 0 ms
- *               1) doc  html [130] unlimited
- *               2) doc  txt  [130] unlimited
- *               3) doc  xhtml [130] unlimited
- *               4) doc  xml  [130] unlimited
- *               5) mpp  html [130] unlimited
- *               6) mpp  txt  [130] unlimited
- *               7) mpp  xhtml [130] unlimited
- *               8) mpp  xml  [130] unlimited
- *               9) msg  html [130] unlimited
- *              10) msg  txt  [130] unlimited
- *              11) msg  xhtml [130] unlimited
- *              12) msg  xml  [130] unlimited
- *              13) ppt  html [130] unlimited
- *              14) ppt  txt  [130] unlimited
- *              15) ppt  xhtml [130] unlimited
- *              16) ppt  xml  [130] unlimited
- *              17) vsd  html [130] unlimited
- *              18) vsd  txt  [130] unlimited
- *              19) vsd  xhtml [130] unlimited
- *              20) vsd  xml  [130] unlimited
- *           Poi 0 ms
- *               1) xls  csv  [130] unlimited
- *               2) xls  html [130] unlimited
- *               3) xls  txt  [130] unlimited
- *               4) xls  xhtml [130] unlimited
- *               5) xls  xml  [130] unlimited
- *               6) xlsx csv  [130] unlimited
- *               7) xlsx html [130] unlimited
- *               8) xlsx txt  [130] unlimited
- *               9) xlsx xhtml [130] unlimited
- *              10) xlsx xml  [130] unlimited
- *           OOXML 0 ms
- *               1) docm html [130] unlimited
- *               2) docm txt  [130] unlimited
- *               3) docm xhtml [130] unlimited
- *               4) docm xml  [130] unlimited
- *               5) docx html [130] unlimited
- *               6) docx txt  [130] unlimited
- *               7) docx xhtml [130] unlimited
- *               8) docx xml  [130] unlimited
- *               9) dotm html [130] unlimited
- *              10) dotm txt  [130] unlimited
- *              11) dotm xhtml [130] unlimited
- *              12) dotm xml  [130] unlimited
- *              13) dotx html [130] unlimited
- *              14) dotx txt  [130] unlimited
- *              15) dotx xhtml [130] unlimited
- *              16) dotx xml  [130] unlimited
- *              17) potm html [130] unlimited
- *              18) potm txt  [130] unlimited
- *              19) potm xhtml [130] unlimited
- *              20) potm xml  [130] unlimited
- *              21) potx html [130] unlimited
- *              22) potx txt  [130] unlimited
- *              23) potx xhtml [130] unlimited
- *              24) potx xml  [130] unlimited
- *              25) ppam html [130] unlimited
- *              26) ppam txt  [130] unlimited
- *              27) ppam xhtml [130] unlimited
- *              28) ppam xml  [130] unlimited
- *              29) ppsm html [130] unlimited
- *              30) ppsm txt  [130] unlimited
- *              31) ppsm xhtml [130] unlimited
- *              32) ppsm xml  [130] unlimited
- *              33) ppsx html [130] unlimited
- *              34) ppsx txt  [130] unlimited
- *              35) ppsx xhtml [130] unlimited
- *              36) ppsx xml  [130] unlimited
- *              37) pptm html [130] unlimited
- *              38) pptm txt  [130] unlimited
- *              39) pptm xhtml [130] unlimited
- *              40) pptm xml  [130] unlimited
- *              41) pptx html [130] unlimited
- *              42) pptx txt  [130] unlimited
- *              43) pptx xhtml [130] unlimited
- *              44) pptx xml  [130] unlimited
- *              45) sldm html [130] unlimited
- *              46) sldm txt  [130] unlimited
- *              47) sldm xhtml [130] unlimited
- *              48) sldm xml  [130] unlimited
- *              49) sldx html [130] unlimited
- *              50) sldx txt  [130] unlimited
- *              51) sldx xhtml [130] unlimited
- *              52) sldx xml  [130] unlimited
- *              53) xlam html [130] unlimited
- *              54) xlam txt  [130] unlimited
- *              55) xlam xhtml [130] unlimited
- *              56) xlam xml  [130] unlimited
- *              57) xlsb html [130] unlimited
- *              58) xlsb txt  [130] unlimited
- *              59) xlsb xhtml [130] unlimited
- *              60) xlsb xml  [130] unlimited
- *              61) xlsm html [130] unlimited
- *              62) xlsm txt  [130] unlimited
- *              63) xlsm xhtml [130] unlimited
- *              64) xlsm xml  [130] unlimited
- *              65) xlsx html [130] unlimited
- *              66) xlsx txt  [130] unlimited
- *              67) xlsx xhtml [130] unlimited
- *              68) xlsx xml  [130] unlimited
- *              69) xltm html [130] unlimited
- *              70) xltm txt  [130] unlimited
- *              71) xltm xhtml [130] unlimited
- *              72) xltm xml  [130] unlimited
- *              73) xltx html [130] unlimited
- *              74) xltx txt  [130] unlimited
- *              75) xltx xhtml [130] unlimited
- *              76) xltx xml  [130] unlimited
- *           TikaAuto 0 ms
- *               1) cdf  html [120] unlimited
- *               2) cdf  txt  [120] unlimited
- *               3) cdf  xhtml [120] unlimited
- *               4) cdf  xml  [120] unlimited
- *               5) cpio html [120] unlimited
- *               6) cpio txt  [120] unlimited
- *               7) cpio xhtml [120] unlimited
- *               8) cpio xml  [120] unlimited
- *               9) doc  html [120] unlimited
- *              10) doc  txt  [120] unlimited
- *              11) doc  xhtml [120] unlimited
- *              12) doc  xml  [120] unlimited
- *              13) docm html [120] unlimited
- *              14) docm txt  [120] unlimited
- *              15) docm xhtml [120] unlimited
- *              16) docm xml  [120] unlimited
- *              17) docx html [120] unlimited
- *              18) docx txt  [120] unlimited
- *              19) docx xhtml [120] unlimited
- *              20) docx xml  [120] unlimited
- *              21) dotm html [120] unlimited
- *              22) dotm txt  [120] unlimited
- *              23) dotm xhtml [120] unlimited
- *              24) dotm xml  [120] unlimited
- *              25) dotx html [120] unlimited
- *              26) dotx txt  [120] unlimited
- *              27) dotx xhtml [120] unlimited
- *              28) dotx xml  [120] unlimited
- *              29) gzip html [120] unlimited
- *              30) gzip txt  [120] unlimited
- *              31) gzip xhtml [120] unlimited
- *              32) gzip xml  [120] unlimited
- *              33) hdf  html [120] unlimited
- *              34) hdf  txt  [120] unlimited
- *              35) hdf  xhtml [120] unlimited
- *              36) hdf  xml  [120] unlimited
- *              37) html html [120] unlimited
- *              38) html txt  [120] unlimited
- *              39) html xhtml [120] unlimited
- *              40) html xml  [120] unlimited
- *              41) jar  html [120] unlimited
- *              42) jar  txt  [120] unlimited
- *              43) jar  xhtml [120] unlimited
- *              44) jar  xml  [120] unlimited
- *              45) java html [120] unlimited
- *              46) java txt  [120] unlimited
- *              47) java xhtml [120] unlimited
- *              48) java xml  [120] unlimited
- *              49) key  html [120] unlimited
- *              50) key  txt  [120] unlimited
- *              51) key  xhtml [120] unlimited
- *              52) key  xml  [120] unlimited
- *              53) mpp  html [120] unlimited
- *              54) mpp  txt  [120] unlimited
- *              55) mpp  xhtml [120] unlimited
- *              56) mpp  xml  [120] unlimited
- *              57) numbers html [120] unlimited
- *              58) numbers txt  [120] unlimited
- *              59) numbers xhtml [120] unlimited
- *              60) numbers xml  [120] unlimited
- *              61) odc  html [120] unlimited
- *              62) odc  txt  [120] unlimited
- *              63) odc  xhtml [120] unlimited
- *              64) odc  xml  [120] unlimited
- *              65) odi  html [120] unlimited
- *              66) odi  txt  [120] unlimited
- *              67) odi  xhtml [120] unlimited
- *              68) odi  xml  [120] unlimited
- *              69) odm  html [120] unlimited
- *              70) odm  txt  [120] unlimited
- *              71) odm  xhtml [120] unlimited
- *              72) odm  xml  [120] unlimited
- *              73) odp  html [120] unlimited
- *              74) odp  txt  [120] unlimited
- *              75) odp  xhtml [120] unlimited
- *              76) odp  xml  [120] unlimited
- *              77) ods  html [120] unlimited
- *              78) ods  txt  [120] unlimited
- *              79) ods  xhtml [120] unlimited
- *              80) ods  xml  [120] unlimited
- *              81) odt  html [120] unlimited
- *              82) odt  txt  [120] unlimited
- *              83) odt  xhtml [120] unlimited
- *              84) odt  xml  [120] unlimited
- *              85) ogx  html [120] unlimited
- *              86) ogx  txt  [120] unlimited
- *              87) ogx  xhtml [120] unlimited
- *              88) ogx  xml  [120] unlimited
- *              89) oth  html [120] unlimited
- *              90) oth  txt  [120] unlimited
- *              91) oth  xhtml [120] unlimited
- *              92) oth  xml  [120] unlimited
- *              93) otp  html [120] unlimited
- *              94) otp  txt  [120] unlimited
- *              95) otp  xhtml [120] unlimited
- *              96) otp  xml  [120] unlimited
- *              97) ots  html [120] unlimited
- *              98) ots  txt  [120] unlimited
- *              99) ots  xhtml [120] unlimited
- *             100) ots  xml  [120] unlimited
- *             101) ott  html [120] unlimited
- *             102) ott  txt  [120] unlimited
- *             103) ott  xhtml [120] unlimited
- *             104) ott  xml  [120] unlimited
- *             105) pages html [120] unlimited
- *             106) pages txt  [120] unlimited
- *             107) pages xhtml [120] unlimited
- *             108) pages xml  [120] unlimited
- *             109) pdf  html [120] unlimited
- *             110) pdf  txt  [120] 25 MB
- *             111) pdf  xhtml [120] unlimited
- *             112) pdf  xml  [120] unlimited
- *             113) potm html [120] unlimited
- *             114) potm txt  [120] unlimited
- *             115) potm xhtml [120] unlimited
- *             116) potm xml  [120] unlimited
- *             117) potx html [120] unlimited
- *             118) potx txt  [120] unlimited
- *             119) potx xhtml [120] unlimited
- *             120) potx xml  [120] unlimited
- *             121) ppam html [120] unlimited
- *             122) ppam txt  [120] unlimited
- *             123) ppam xhtml [120] unlimited
- *             124) ppam xml  [120] unlimited
- *             125) ppsm html [120] unlimited
- *             126) ppsm txt  [120] unlimited
- *             127) ppsm xhtml [120] unlimited
- *             128) ppsm xml  [120] unlimited
- *             129) ppsx html [120] unlimited
- *             130) ppsx txt  [120] unlimited
- *             131) ppsx xhtml [120] unlimited
- *             132) ppsx xml  [120] unlimited
- *             133) ppt  html [120] unlimited
- *             134) ppt  txt  [120] unlimited
- *             135) ppt  xhtml [120] unlimited
- *             136) ppt  xml  [120] unlimited
- *             137) pptm html [120] unlimited
- *             138) pptm txt  [120] unlimited
- *             139) pptm xhtml [120] unlimited
- *             140) pptm xml  [120] unlimited
- *             141) pptx html [120] unlimited
- *             142) pptx txt  [120] unlimited
- *             143) pptx xhtml [120] unlimited
- *             144) pptx xml  [120] unlimited
- *             145) rar  html [120] unlimited
- *             146) rar  txt  [120] unlimited
- *             147) rar  xhtml [120] unlimited
- *             148) rar  xml  [120] unlimited
- *             149) rss  html [120] unlimited
- *             150) rss  txt  [120] unlimited
- *             151) rss  xhtml [120] unlimited
- *             152) rss  xml  [120] unlimited
- *             153) rtf  html [120] unlimited
- *             154) rtf  txt  [120] unlimited
- *             155) rtf  xhtml [120] unlimited
- *             156) rtf  xml  [120] unlimited
- *             157) sldm html [120] unlimited
- *             158) sldm txt  [120] unlimited
- *             159) sldm xhtml [120] unlimited
- *             160) sldm xml  [120] unlimited
- *             161) sldx html [120] unlimited
- *             162) sldx txt  [120] unlimited
- *             163) sldx xhtml [120] unlimited
- *             164) sldx xml  [120] unlimited
- *             165) sxw  html [120] unlimited
- *             166) sxw  txt  [120] unlimited
- *             167) sxw  xhtml [120] unlimited
- *             168) sxw  xml  [120] unlimited
- *             169) txt  html [120] unlimited
- *             170) txt  txt  [120] unlimited
- *             171) txt  xhtml [120] unlimited
- *             172) txt  xml  [120] unlimited
- *             173) vsd  html [120] unlimited
- *             174) vsd  txt  [120] unlimited
- *             175) vsd  xhtml [120] unlimited
- *             176) vsd  xml  [120] unlimited
- *             177) xhtml html [120] unlimited
- *             178) xhtml txt  [120] unlimited
- *             179) xhtml xhtml [120] unlimited
- *             180) xhtml xml  [120] unlimited
- *             181) xlam html [120] unlimited
- *             182) xlam txt  [120] unlimited
- *             183) xlam xhtml [120] unlimited
- *             184) xlam xml  [120] unlimited
- *             185) xls  html [120] unlimited
- *             186) xls  txt  [120] unlimited
- *             187) xls  xhtml [120] unlimited
- *             188) xls  xml  [120] unlimited
- *             189) xlsb html [120] unlimited
- *             190) xlsb txt  [120] unlimited
- *             191) xlsb xhtml [120] unlimited
- *             192) xlsb xml  [120] unlimited
- *             193) xlsm html [120] unlimited
- *             194) xlsm txt  [120] unlimited
- *             195) xlsm xhtml [120] unlimited
- *             196) xlsm xml  [120] unlimited
- *             197) xlsx html [120] unlimited
- *             198) xlsx txt  [120] unlimited
- *             199) xlsx xhtml [120] unlimited
- *             200) xlsx xml  [120] unlimited
- *             201) xltm html [120] unlimited
- *             202) xltm txt  [120] unlimited
- *             203) xltm xhtml [120] unlimited
- *             204) xltm xml  [120] unlimited
- *             205) xltx html [120] unlimited
- *             206) xltx txt  [120] unlimited
- *             207) xltx xhtml [120] unlimited
- *             208) xltx xml  [120] unlimited
- *             209) xml  html [120] unlimited
- *             210) xml  txt  [120] unlimited
- *             211) xml  xhtml [120] unlimited
- *             212) xml  xml  [120] unlimited
- *             213) z    html [120] unlimited
- *             214) z    txt  [120] unlimited
- *             215) z    xhtml [120] unlimited
- *             216) z    xml  [120] unlimited
- *           TextMining 0 ms
- *               1) doc  html [130] unlimited
- *               2) doc  txt   [50] unlimited
- *               3) doc  xhtml [130] unlimited
- *               4) doc  xml  [130] unlimited
- *
- */ -public class Tika -{ - public static final String ARCHIVE = "Archive"; - public static final String OUTLOOK_MSG = "OutlookMsg"; - public static final String PDF_BOX = "PdfBox"; - public static final String POI_OFFICE = "Office"; - public static final String POI = "Poi"; - public static final String POI_OO_XML = "OOXML"; - public static final String TIKA_AUTO = "TikaAuto"; - public static final String TEXT_MINING = "TextMining"; - - public static final List TRANSFORM_NAMES = ImmutableList.of( - ARCHIVE, OUTLOOK_MSG, PDF_BOX, POI_OFFICE, POI, POI_OO_XML, TIKA_AUTO, TEXT_MINING); - - public static final String TARGET_MIMETYPE = "--targetMimetype="; - public static final String TARGET_ENCODING = "--targetEncoding="; - public static final String INCLUDE_CONTENTS = "--includeContents"; - public static final String NOT_EXTRACT_BOOKMARKS_TEXT = "--notExtractBookmarksText"; - - public static final String CSV = "csv"; - public static final String DOC = "doc"; - public static final String DOCX = "docx"; - public static final String HTML = "html"; - public static final String MSG = "msg"; - public static final String PDF = "pdf"; - public static final String PPTX = "pptx"; - public static final String TXT = "txt"; - public static final String XHTML = "xhtml"; - public static final String XSLX = "xslx"; - public static final String XML = "xml"; - public static final String ZIP = "zip"; - - private final Parser packageParser = new PackageParser(); - private final Parser pdfParser = new PDFParser(); - private final Parser officeParser = new OfficeParser(); - private final Parser autoDetectParser; - private final Parser ooXmlParser = new OOXMLParser(); - private final Parser tikaOfficeDetectParser = new TikaOfficeDetectParser(); - private final PDFParserConfig pdfParserConfig = new PDFParserConfig(); - - public static final DocumentSelector pdfBoxEmbededDocumentSelector = new DocumentSelector() - { - private final List disabledMediaTypes = ImmutableList.of(MIMETYPE_IMAGE_JPEG, - MIMETYPE_IMAGE_TIFF, MIMETYPE_IMAGE_PNG); - - @Override - public boolean select(Metadata metadata) - { - String contentType = metadata.get(Metadata.CONTENT_TYPE); - if (contentType == null || contentType.equals("") || disabledMediaTypes == null) - { - return true; - } - return !disabledMediaTypes.contains(contentType); - } - }; - - public Tika() throws TikaException, IOException, SAXException - { - TikaConfig tikaConfig = readTikaConfig(); - autoDetectParser = new AutoDetectParser(tikaConfig); - } - - public static TikaConfig readTikaConfig(Logger logger) - { - try - { - return readTikaConfig(); - } - catch (Exception e) - { - logger.error("Failed to read tika-config.xml", e); - return null; - } - } - - private static TikaConfig readTikaConfig() throws TikaException, IOException, SAXException - { - ClassLoader classLoader = Tika.class.getClassLoader(); - URL tikaConfigXml = classLoader.getResource("tika-config.xml"); - return new TikaConfig(tikaConfigXml); - } - - // Method included for developer testing - public static void main(String[] args) - { - long start = System.currentTimeMillis(); - try - { - new Tika().transform(args); - } - catch (IllegalArgumentException e) - { - System.err.println("ERROR " + e.getMessage()); - System.exit(-1); - } - catch (IllegalStateException | TikaException | IOException | SAXException e) - { - System.err.println("ERROR " + e.getMessage()); - e.printStackTrace(); - System.exit(-2); - } - System.out.println("Finished in " + (System.currentTimeMillis() - start) + "ms"); - } - - // Extracts parameters form args - public void transform(String[] args) - { - String transform = null; - String targetMimetype = null; - String targetEncoding = null; - String sourceFilename = null; - String targetFilename = null; - Boolean includeContents = null; - Boolean notExtractBookmarksText = null; - - for (String arg : args) - { - if (arg.startsWith("--")) - { - if (INCLUDE_CONTENTS.startsWith(arg)) - { - getValue(arg, false, includeContents, INCLUDE_CONTENTS); - includeContents = true; - } - else if (arg.startsWith(TARGET_ENCODING)) - { - targetEncoding = getValue(arg, true, targetEncoding, TARGET_ENCODING); - } - else if (arg.startsWith(TARGET_MIMETYPE)) - { - targetMimetype = getValue(arg, true, targetMimetype, TARGET_MIMETYPE); - } - else if (arg.startsWith(NOT_EXTRACT_BOOKMARKS_TEXT)) - { - getValue(arg, false, notExtractBookmarksText, NOT_EXTRACT_BOOKMARKS_TEXT); - notExtractBookmarksText = true; - } - else - { - throw new IllegalArgumentException("Unexpected argument " + arg); - } - } - else - { - if (transform == null) - { - transform = arg; - } - else if (sourceFilename == null) - { - sourceFilename = arg; - } - else if (targetFilename == null) - { - targetFilename = arg; - } - else - { - throw new IllegalArgumentException("Unexpected argument " + arg); - } - } - } - if (targetFilename == null) - { - throw new IllegalArgumentException("Missing arguments"); - } - includeContents = includeContents == null ? false : includeContents; - notExtractBookmarksText = notExtractBookmarksText == null ? false : notExtractBookmarksText; - - transform(transform, includeContents, notExtractBookmarksText, sourceFilename, - targetFilename, targetMimetype, targetEncoding); - } - - private String getValue(String arg, boolean valueExpected, Object value, String optionName) - { - if (value != null) - { - throw new IllegalArgumentException("Duplicate " + optionName); - } - String stringValue = arg.substring(optionName.length()).trim(); - if (!valueExpected && stringValue.length() > 0) - { - throw new IllegalArgumentException("Unexpected value with " + optionName); - } - if (valueExpected && stringValue.length() == 0) - { - throw new IllegalArgumentException("Expected value with " + optionName); - } - return stringValue; - } - - // Adds transform specific values such as parser and documentSelector. - private void transform(String transform, Boolean includeContents, - Boolean notExtractBookmarksText, - String sourceFilename, - String targetFilename, String targetMimetype, String targetEncoding) - { - Parser parser = null; - DocumentSelector documentSelector = null; - - switch (transform) - { - case ARCHIVE: - parser = packageParser; - break; - case OUTLOOK_MSG: - case POI_OFFICE: - case TEXT_MINING: - parser = officeParser; - break; - case PDF_BOX: - parser = pdfParser; - documentSelector = pdfBoxEmbededDocumentSelector; - break; - case POI: - parser = tikaOfficeDetectParser; - break; - case POI_OO_XML: - parser = ooXmlParser; - break; - case TIKA_AUTO: - parser = autoDetectParser; - break; - } - - transform(parser, documentSelector, includeContents, notExtractBookmarksText, - sourceFilename, targetFilename, targetMimetype, targetEncoding); - } - - private void transform(Parser parser, DocumentSelector documentSelector, - Boolean includeContents, - Boolean notExtractBookmarksText, - String sourceFilename, - String targetFilename, String targetMimetype, String targetEncoding) - { - - try (InputStream is = new BufferedInputStream(new FileInputStream(sourceFilename)); - OutputStream os = new FileOutputStream(targetFilename); - Writer ow = new BufferedWriter(new OutputStreamWriter(os, targetEncoding))) - { - Metadata metadata = new Metadata(); - ParseContext context = buildParseContext(documentSelector, includeContents, - notExtractBookmarksText); - ContentHandler handler = getContentHandler(targetMimetype, ow); - - parser.parse(is, handler, metadata, context); - } - catch (SAXException | TikaException | IOException e) - { - throw new IllegalStateException(e.getMessage(), e); - } - } - - private ContentHandler getContentHandler(String targetMimetype, Writer output) - { - try - { - ContentHandler handler; - if (MIMETYPE_TEXT_PLAIN.equals(targetMimetype)) - { - handler = new BodyContentHandler(output); - } - else - { - SAXTransformerFactory factory = (SAXTransformerFactory) SAXTransformerFactory.newInstance(); - TransformerHandler transformerHandler; - transformerHandler = factory.newTransformerHandler(); - transformerHandler.getTransformer().setOutputProperty(OutputKeys.INDENT, "yes"); - transformerHandler.setResult(new StreamResult(output)); - handler = transformerHandler; - - if (MIMETYPE_HTML.equals(targetMimetype)) - { - transformerHandler.getTransformer().setOutputProperty(OutputKeys.METHOD, HTML); - return new ExpandedTitleContentHandler(transformerHandler); - } - else if (MIMETYPE_XHTML.equals(targetMimetype) || - MIMETYPE_XML.equals(targetMimetype)) - { - transformerHandler.getTransformer().setOutputProperty(OutputKeys.METHOD, XML); - } - else if (MIMETYPE_TEXT_CSV.equals(targetMimetype)) - { - handler = new CsvContentHandler(output); - } - else - { - throw new IllegalArgumentException("Invalid target mimetype " + targetMimetype); - } - } - return handler; - } - catch (TransformerConfigurationException e) - { - throw new IllegalStateException(e.getMessage(), e); - } - } - - /** - * A wrapper around the normal Tika BodyContentHandler for CSV rather encoding than tab separated. - */ - protected static class CsvContentHandler extends BodyContentHandler - { - private static final char[] comma = new char[]{','}; - private static final Pattern all_nums = Pattern.compile("[\\d\\.\\-\\+]+"); - - private boolean inCell = false; - private boolean needsComma = false; - - protected CsvContentHandler(Writer output) - { - super(output); - } - - @Override - public void ignorableWhitespace(char[] ch, int start, int length) - throws SAXException - { - if (length == 1 && ch[0] == '\t') - { - // Ignore tabs, as they mess up the CSV output - } - else - { - super.ignorableWhitespace(ch, start, length); - } - } - - @Override - public void characters(char[] ch, int start, int length) - throws SAXException - { - if (inCell) - { - StringBuffer t = new StringBuffer(new String(ch, start, length)); - - // Quote if not all numbers - if (all_nums.matcher(t).matches()) - { - super.characters(ch, start, length); - } - else - { - for (int i = t.length() - 1; i >= 0; i--) - { - if (t.charAt(i) == '\"') - { - // Double up double quotes - t.insert(i, '\"'); - i--; - } - } - t.insert(0, '\"'); - t.append('\"'); - char[] c = t.toString().toCharArray(); - super.characters(c, 0, c.length); - } - } - else - { - super.characters(ch, start, length); - } - } - - @Override - public void startElement(String uri, String localName, String name, - Attributes atts) throws SAXException - { - if (localName.equals("td")) - { - inCell = true; - if (needsComma) - { - super.characters(comma, 0, 1); - needsComma = true; - } - } - else - { - super.startElement(uri, localName, name, atts); - } - } - - @Override - public void endElement(String uri, String localName, String name) - throws SAXException - { - if (localName.equals("td")) - { - needsComma = true; - inCell = false; - } - else - { - if (localName.equals("tr")) - { - needsComma = false; - } - super.endElement(uri, localName, name); - } - } - } - - private ParseContext buildParseContext(DocumentSelector documentSelector, - Boolean includeContents, Boolean notExtractBookmarksText) - { - ParseContext context = new ParseContext(); - - if (documentSelector != null) - { - context.set(DocumentSelector.class, documentSelector); - } - - if (notExtractBookmarksText.equals(true)) - { - pdfParserConfig.setExtractBookmarksText(false); - // pdfParserConfig is set to override default settings - context.set(PDFParserConfig.class, pdfParserConfig); - } - - // If Archive transform - if (includeContents != null) - { - context.set(Parser.class, includeContents ? autoDetectParser : new EmptyParser()); - } - - return context; - } -} diff --git a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/executors/TikaJavaExecutor.java b/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/executors/TikaJavaExecutor.java deleted file mode 100644 index 7820662a..00000000 --- a/alfresco-transform-tika/alfresco-transform-tika/src/main/java/org/alfresco/transformer/executors/TikaJavaExecutor.java +++ /dev/null @@ -1,204 +0,0 @@ -/* - * #%L - * Alfresco Transform Core - * %% - * Copyright (C) 2005 - 2021 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * - - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ -package org.alfresco.transformer.executors; - -import com.google.common.collect.ImmutableMap; -import org.alfresco.transformer.logging.LogEntry; -import org.alfresco.transformer.metadataExtractors.AbstractTikaMetadataExtractor; -import org.alfresco.transformer.metadataExtractors.DWGMetadataExtractor; -import org.alfresco.transformer.metadataExtractors.MP3MetadataExtractor; -import org.alfresco.transformer.metadataExtractors.MailMetadataExtractor; -import org.alfresco.transformer.metadataExtractors.OfficeMetadataExtractor; -import org.alfresco.transformer.metadataExtractors.OpenDocumentMetadataExtractor; -import org.alfresco.transformer.metadataExtractors.PdfBoxMetadataExtractor; -import org.alfresco.transformer.metadataExtractors.PoiMetadataExtractor; -import org.alfresco.transformer.metadataExtractors.TikaAudioMetadataExtractor; -import org.alfresco.transformer.metadataExtractors.TikaAutoMetadataExtractor; -import org.alfresco.transformer.metadataExtractors.IPTCMetadataExtractor; -import org.alfresco.transformer.util.RequestParamMap; -import org.apache.tika.exception.TikaException; -import org.slf4j.LoggerFactory; -import org.xml.sax.SAXException; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Map; -import java.util.StringJoiner; - -import static java.lang.Boolean.parseBoolean; -import static org.alfresco.transformer.executors.Tika.INCLUDE_CONTENTS; -import static org.alfresco.transformer.executors.Tika.TARGET_ENCODING; -import static org.alfresco.transformer.executors.Tika.TARGET_MIMETYPE; - -/** - * JavaExecutor implementation for running TIKA transformations. It loads the - * transformation logic in the same JVM (check {@link Tika}). - */ -public class TikaJavaExecutor implements JavaExecutor -{ - private boolean notExtractBookmarksTextDefault; - - private static final String ID = "tika"; - - public static final String LICENCE = - "This transformer uses Tika from Apache. See the license at http://www.apache.org/licenses/LICENSE-2.0. or in /Apache\\ 2.0.txt\n" + - "This transformer uses ExifTool by Phil Harvey. See license at https://exiftool.org/#license. or in /Perl-Artistic-License.txt"; - - private final Tika tika; - private final Map metadataExtractor = ImmutableMap - .builder() - .put("DWGMetadataExtractor", new DWGMetadataExtractor()) - .put("MailMetadataExtractor", new MailMetadataExtractor()) - .put("MP3MetadataExtractor", new MP3MetadataExtractor()) - .put("OfficeMetadataExtractor", new OfficeMetadataExtractor()) - .put("OpenDocumentMetadataExtractor", new OpenDocumentMetadataExtractor()) - .put("PdfBoxMetadataExtractor", new PdfBoxMetadataExtractor()) - .put("PoiMetadataExtractor", new PoiMetadataExtractor()) - .put("TikaAudioMetadataExtractor", new TikaAudioMetadataExtractor()) - .put("TikaAutoMetadataExtractor", new TikaAutoMetadataExtractor()) - .put("IPTCMetadataExtractor", new IPTCMetadataExtractor()) - .build(); - private final Map metadataEmbedder = ImmutableMap - .builder() - .put("SamplePoiMetadataEmbedder", new PoiMetadataExtractor()) - .build(); - - public TikaJavaExecutor(boolean notExtractBookmarksTextDefault) - { - this.notExtractBookmarksTextDefault = notExtractBookmarksTextDefault; - try - { - tika = new Tika(); - } - catch (SAXException | IOException | TikaException e) - { - throw new RuntimeException("Unable to instantiate Tika: " + e.getMessage()); - } - } - - public TikaJavaExecutor() - { - this(false); - } - - @Override - public String getTransformerId() - { - return ID; - } - - @Override - public void transform(String transformName, String sourceMimetype, String targetMimetype, - Map transformOptions, File sourceFile, File targetFile) - throws Exception - { - final boolean includeContents = parseBoolean( - transformOptions.getOrDefault(RequestParamMap.INCLUDE_CONTENTS, "false")); - final boolean notExtractBookmarksText = parseBoolean( - transformOptions.getOrDefault(RequestParamMap.NOT_EXTRACT_BOOKMARKS_TEXT, String.valueOf(notExtractBookmarksTextDefault))); - final String targetEncoding = transformOptions.getOrDefault("targetEncoding", "UTF-8"); - if(transformOptions.get(RequestParamMap.NOT_EXTRACT_BOOKMARKS_TEXT)==null && notExtractBookmarksTextDefault) - { - LoggerFactory.getLogger(TikaJavaExecutor.class).trace( - "notExtractBookmarksText default value has been overridden to {}", - notExtractBookmarksTextDefault); - } - call(sourceFile, targetFile, transformName, - includeContents ? INCLUDE_CONTENTS : null, - notExtractBookmarksText ? Tika.NOT_EXTRACT_BOOKMARKS_TEXT : null, - TARGET_MIMETYPE + targetMimetype, TARGET_ENCODING + targetEncoding); - } - - @Override - public void call(File sourceFile, File targetFile, String... args) - { - args = buildArgs(sourceFile, targetFile, args); - tika.transform(args); - } - - private static String[] buildArgs(File sourceFile, File targetFile, String[] args) - { - ArrayList methodArgs = new ArrayList<>(args.length + 2); - StringJoiner sj = new StringJoiner(" "); - for (String arg : args) - { - addArg(methodArgs, sj, arg); - } - - addFileArg(methodArgs, sj, sourceFile); - addFileArg(methodArgs, sj, targetFile); - - LogEntry.setOptions(sj.toString()); - - return methodArgs.toArray(new String[0]); - } - - private static void addArg(ArrayList methodArgs, StringJoiner sj, String arg) - { - if (arg != null) - { - sj.add(arg); - methodArgs.add(arg); - } - } - - private static void addFileArg(ArrayList methodArgs, StringJoiner sj, File arg) - { - if (arg != null) - { - String path = arg.getAbsolutePath(); - int i = path.lastIndexOf('.'); - String ext = i == -1 ? "???" : path.substring(i + 1); - sj.add(ext); - methodArgs.add(path); - } - } - - public void extractMetadata(String transformName, String sourceMimetype, String targetMimetype, - Map transformOptions, File sourceFile, File targetFile) - throws Exception - { - AbstractTikaMetadataExtractor metadataExtractor = this.metadataExtractor.get(transformName); - metadataExtractor.extractMetadata(sourceMimetype, transformOptions, sourceFile, targetFile); - } - - /** - * @deprecated The content repository's TikaPoweredMetadataExtracter provides no non test implementations. - * This code exists in case there are custom implementations, that need to be converted to T-Engines. - * It is simply a copy and paste from the content repository and has received limited testing. - */ - @Override - @SuppressWarnings("deprecation" ) - public void embedMetadata(String transformName, String sourceMimetype, String targetMimetype, - Map transformOptions, File sourceFile, File targetFile) - throws Exception - { - AbstractTikaMetadataExtractor metadataExtractor = this.metadataEmbedder.get(transformName); - metadataExtractor.embedMetadata(sourceMimetype, targetMimetype, transformOptions, sourceFile, targetFile); - } -} diff --git a/alfresco-transform-tika/alfresco-transform-tika/src/test/java/org/alfresco/transformer/metadataExtractors/IPTCMetadataExtractorTest.java b/alfresco-transform-tika/alfresco-transform-tika/src/test/java/org/alfresco/transform/tika/metadataExtractors/IPTCMetadataExtractorTest.java similarity index 91% rename from alfresco-transform-tika/alfresco-transform-tika/src/test/java/org/alfresco/transformer/metadataExtractors/IPTCMetadataExtractorTest.java rename to alfresco-transform-tika/alfresco-transform-tika/src/test/java/org/alfresco/transform/tika/metadataExtractors/IPTCMetadataExtractorTest.java index fc35dbcf..b16a5df8 100644 --- a/alfresco-transform-tika/alfresco-transform-tika/src/test/java/org/alfresco/transformer/metadataExtractors/IPTCMetadataExtractorTest.java +++ b/alfresco-transform-tika/alfresco-transform-tika/src/test/java/org/alfresco/transform/tika/metadataExtractors/IPTCMetadataExtractorTest.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2021 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,14 +24,14 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer.metadataExtractors; +package org.alfresco.transform.tika.metadataExtractors; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import org.junit.jupiter.api.Test; -public class IPTCMetadataExtractorTest { - +public class IPTCMetadataExtractorTest +{ IPTCMetadataExtractor extractor = new IPTCMetadataExtractor(); @Test @@ -42,7 +42,5 @@ public class IPTCMetadataExtractorTest { "1901-02-01T00:00:00.000Z", "1901-02-01T00:00+00:00", "1901-02-01T00:00+00:00" }; assertArrayEquals(expected, extractor.iptcToIso8601DateStrings(testStrings)); - } - } diff --git a/alfresco-transform-tika/alfresco-transform-tika/src/test/java/org/alfresco/transformer/tika/parsers/ExifToolParserTest.java b/alfresco-transform-tika/alfresco-transform-tika/src/test/java/org/alfresco/transform/tika/parsers/ExifToolParserTest.java similarity index 92% rename from alfresco-transform-tika/alfresco-transform-tika/src/test/java/org/alfresco/transformer/tika/parsers/ExifToolParserTest.java rename to alfresco-transform-tika/alfresco-transform-tika/src/test/java/org/alfresco/transform/tika/parsers/ExifToolParserTest.java index 4303f255..8b822ecd 100644 --- a/alfresco-transform-tika/alfresco-transform-tika/src/test/java/org/alfresco/transformer/tika/parsers/ExifToolParserTest.java +++ b/alfresco-transform-tika/alfresco-transform-tika/src/test/java/org/alfresco/transform/tika/parsers/ExifToolParserTest.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2021 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,7 +24,7 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer.tika.parsers; +package org.alfresco.transform.tika.parsers; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/alfresco-transform-tika/alfresco-transform-tika/src/test/java/org/alfresco/transformer/executors/TikaJavaExecutorTest.java b/alfresco-transform-tika/alfresco-transform-tika/src/test/java/org/alfresco/transform/tika/transformers/GenericTikaTransformerTest.java similarity index 84% rename from alfresco-transform-tika/alfresco-transform-tika/src/test/java/org/alfresco/transformer/executors/TikaJavaExecutorTest.java rename to alfresco-transform-tika/alfresco-transform-tika/src/test/java/org/alfresco/transform/tika/transformers/GenericTikaTransformerTest.java index 509a2be7..4774321b 100644 --- a/alfresco-transform-tika/alfresco-transform-tika/src/test/java/org/alfresco/transformer/executors/TikaJavaExecutorTest.java +++ b/alfresco-transform-tika/alfresco-transform-tika/src/test/java/org/alfresco/transform/tika/transformers/GenericTikaTransformerTest.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2021 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,11 +24,11 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer.executors; +package org.alfresco.transform.tika.transformers; -import static org.alfresco.transformer.executors.Tika.NOT_EXTRACT_BOOKMARKS_TEXT; -import static org.alfresco.transformer.executors.Tika.TARGET_ENCODING; -import static org.alfresco.transformer.executors.Tika.TARGET_MIMETYPE; +import static org.alfresco.transform.tika.transformers.Tika.NOT_EXTRACT_BOOKMARKS_TEXT; +import static org.alfresco.transform.tika.transformers.Tika.TARGET_ENCODING; +import static org.alfresco.transform.tika.transformers.Tika.TARGET_MIMETYPE; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.clearInvocations; import static org.mockito.Mockito.lenient; @@ -41,19 +41,33 @@ import java.io.File; import java.util.HashMap; import java.util.Map; +import org.apache.tika.parser.Parser; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) -public class TikaJavaExecutorTest { +public class GenericTikaTransformerTest +{ + private class TikaTestTransformer extends GenericTikaTransformer + { + @Override + protected Parser getParser() + { + return null; + } + TikaTestTransformer(boolean notExtractBookmarksTextDefault) + { + this.notExtractBookmarksTextDefault = notExtractBookmarksTextDefault; + } + }; @Test public void testNotExtractBookmarkTextDefault() throws Exception { - TikaJavaExecutor executorSpyDefaultTrue = spy(new TikaJavaExecutor(true)); - TikaJavaExecutor executorSpyDefaultFalse = spy(new TikaJavaExecutor(false)); + GenericTikaTransformer executorSpyDefaultTrue = spy(new TikaTestTransformer(true)); + GenericTikaTransformer executorSpyDefaultFalse = spy(new TikaTestTransformer(false)); File mockSourceFile = mock(File.class); File mockTargetFile = mock(File.class); @@ -66,7 +80,7 @@ public class TikaJavaExecutorTest { lenient().doNothing().when(executorSpyDefaultTrue).call(any(), any(), any(), any(), any(), any(), any()); lenient().doNothing().when(executorSpyDefaultFalse).call(any(), any(), any(), any(), any(), any(), any()); - Map transformOptions = new HashMap(); + Map transformOptions = new HashMap<>(); // use empty transformOptions to test defaults executorSpyDefaultTrue.transform(transformName, sourceMimetype, targetMimetype, transformOptions, diff --git a/alfresco-transformer-base/src/main/java/org/alfresco/transformer/util/Util.java b/alfresco-transformer-base/src/main/java/org/alfresco/transformer/util/Util.java index ead426a0..ed36a98a 100644 --- a/alfresco-transformer-base/src/main/java/org/alfresco/transformer/util/Util.java +++ b/alfresco-transformer-base/src/main/java/org/alfresco/transformer/util/Util.java @@ -26,9 +26,6 @@ */ package org.alfresco.transformer.util; -/** - * - */ public class Util { /** diff --git a/pom.xml b/pom.xml index 58b749a5..90671a1c 100644 --- a/pom.xml +++ b/pom.xml @@ -43,6 +43,7 @@ alfresco-transform-model alfresco-transformer-base + t-engine-base alfresco-transform-imagemagick/alfresco-transform-imagemagick alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot alfresco-transform-libreoffice/alfresco-transform-libreoffice @@ -62,6 +63,7 @@ alfresco-transform-model alfresco-transformer-base + t-engine-base @@ -69,6 +71,7 @@ alfresco-transform-model alfresco-transformer-base + t-engine-base alfresco-transform-imagemagick/alfresco-transform-imagemagick alfresco-transform-imagemagick/alfresco-transform-imagemagick-boot @@ -78,6 +81,7 @@ alfresco-transform-model alfresco-transformer-base + t-engine-base alfresco-transform-libreoffice/alfresco-transform-libreoffice alfresco-transform-libreoffice/alfresco-transform-libreoffice-boot @@ -87,6 +91,7 @@ alfresco-transform-model alfresco-transformer-base + t-engine-base alfresco-transform-misc/alfresco-transform-misc alfresco-transform-misc/alfresco-transform-misc-boot @@ -96,6 +101,7 @@ alfresco-transform-model alfresco-transformer-base + t-engine-base alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer alfresco-transform-pdf-renderer/alfresco-transform-pdf-renderer-boot @@ -105,6 +111,7 @@ alfresco-transform-model alfresco-transformer-base + t-engine-base alfresco-transform-tika/alfresco-transform-tika alfresco-transform-tika/alfresco-transform-tika-boot @@ -115,6 +122,7 @@ alfresco-transform-model alfresco-transformer-base + t-engine-base alfresco-transform-core-aio/alfresco-transform-core-aio alfresco-transform-core-aio/alfresco-transform-core-aio-boot diff --git a/t-engine-base/README.md b/t-engine-base/README.md new file mode 100644 index 00000000..e760db33 --- /dev/null +++ b/t-engine-base/README.md @@ -0,0 +1,233 @@ +# Common code for Transform Engines + +This project contains code that is common between all the ACS T-Engine transformers that run as Spring Boot process (optionally within their own +Docker containers). It performs common actions such as logging, throttling requests and handling the streaming of content to and from the container. + +For more details on build a custom T-Engine, please refer to the current docs in ACS Packaging, including: + +* [ATS Configuration](https://github.com/Alfresco/acs-packaging/blob/master/docs/custom-transforms-and-renditions.md#ats-configuration) +* [Creating a T-Engine](https://github.com/Alfresco/acs-packaging/blob/master/docs/creating-a-t-engine.md) + +## Overview + +A transformer project is expected to provide the following files: + +~~~ +src/main/resources/templates/transformForm.html +src/main/java/org/alfresco/transformer/Controller.java +src/main/java/org/alfresco/transformer/Application.java +~~~ + +* transformForm.html - A simple test page using [thymeleaf](http://www.thymeleaf.org) that gathers request + parameters so they may be used to test the transformer. + +~~~ + + +
+

Test Transformation

+
+ + + + + + + + + + + +
file *
file *
sourceExtension *
targetExtension *
sourceMimetype *
targetMimetype *
abc:width
abc:height
timeout
+
+
+ + + +~~~ + +* *TransformerName*Controller.java - A [Spring Boot](https://projects.spring.io/spring-boot/) Controller that + extends AbstractTransformerController to handel requests. It implements a few methods including *transformImpl* + which is intended to perform the actual transform. Generally the transform is done in a sub class of + *JavaExecutor*, when a Java library is being used or *AbstractCommandExecutor*, when an external process is used. + Both are sub interfaces of *Transformer*. + +~~~ +... +@Controller +public class TransformerNameController extends AbstractTransformerController +{ + private static final Logger logger = LoggerFactory.getLogger(TransformerNameController.class); + + TransformerNameExecutor executor; + + @PostConstruct + private void init() + { + executor = new TransformerNameExecutor(); + } + + @Override + public String getTransformerName() + { + return "Transformer Name"; + } + + @Override + public String version() + { + return commandExecutor.version(); + } + + @Override + public ProbeTestTransform getProbeTestTransform() + { + // See the Javadoc on this method and Probes.md for the choice of these values. + return new ProbeTestTransform(this, "quick.pdf", "quick.png", + 7455, 1024, 150, 10240, 60 * 20 + 1, 60 * 15 - 15) + { + @Override + protected void executeTransformCommand(File sourceFile, File targetFile) + { + transformImpl(null, null, null, Collections.emptyMap(), sourceFile, targetFile); + } + }; + } + + @Override + public void transformImpl(String transformName, String sourceMimetype, String targetMimetype, + Map transformOptions, File sourceFile, File targetFile) + { + executor.transform(sourceMimetype, targetMimetype, transformOptions, sourceFile, targetFile); + } +} +~~~ + +* *TransformerName*Executer.java - *JavaExecutor* and *CommandExecutor* sub classes need to extract values from + *transformOptions* and use them in a call to an external process or as parameters to a library call. +~~~ +... +public class TransformerNameExecutor extends AbstractCommandExecutor +{ + ... + @Override + public void transform(String transformName, String sourceMimetype, String targetMimetype, + Map transformOptions, + File sourceFile, File targetFile) throws TransformException + { + final String options = TransformerNameOptionsBuilder + .builder() + .withWidth(transformOptions.get(WIDTH_REQUEST_PARAM)) + .withHeight(transformOptions.get(HEIGHT_REQUEST_PARAM)) + .build(); + + Long timeout = stringToLong(transformOptions.get(TIMEOUT)); + + run(options, sourceFile, targetFile, timeout); + } +} +~~~ + +* Application.java - [Spring Boot](https://projects.spring.io/spring-boot/) expects to find an Application in + a project's source files. The following may be used: + +~~~ +package org.alfresco.transformer; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class Application +{ + public static void main(String[] args) + { + SpringApplication.run(Application.class, args); + } +} +~~~ + +Transform requests are handled by the *AbstractTransformerController*, but are either: + * POST requests (a direct http request from a client) where the transform options are passed as parameters, the source is supplied as a multipart file and + the response is a file download. + * POST request (a request via a message queue) where the transform options are supplied as JSON and the response is also JSON. + The source and target content is read from a location accessible to both the client and the transfomer. + +**Example JSON request body** +```javascript +var transformRequest = { + "requestId": "1", + "sourceReference": "2f9ed237-c734-4366-8c8b-6001819169a4", + "sourceMediaType": "application/pdf", + "sourceSize": 123456, + "sourceExtension": "pdf", + "targetMediaType": "text/plain", + "targetExtension": "txt", + "clientType": "ACS", + "clientData": "Yo No Soy Marinero, Soy Capitan, Soy Capitan!", + "schema": 1, + "transformRequestOptions": { + "targetMimetype": "text/plain", + "targetEncoding": "UTF-8", + "abc:width": "120", + "abc:height": "200" + } +} +``` + +**Example JSON response body** + +```javascript +var transformReply = { + "requestId": "1", + "status": 201, + "errorDetails": null, + "sourceReference": "2f9ed237-c734-4366-8c8b-6001819169a4", + "targetReference": "34d69ff0-7eaa-4741-8a9f-e1915e6995bf", + "clientType": "ACS", + "clientData": "Yo No Soy Marinero, Soy Capitan, Soy Capitan!", + "schema": 1 +} +``` + +## Building and testing + +The project can be built by running the Maven command: + +~~~ +mvn clean install +~~~ + +## Artifacts + +The artifacts can be obtained by: + +* downloading from the [Alfresco repository](https://artifacts.alfresco.com/nexus/content/groups/public/) +* Adding a Maven dependency to your pom file. + +~~~ + + org.alfresco + alfresco-t-engine-base + 1.0 + +~~~ + +and the Alfresco Maven repository: + +~~~ + + alfresco-maven-repo + https://artifacts.alfresco.com/nexus/content/groups/public + +~~~ + +The build plan is available in [TravisCI](https://travis-ci.com/Alfresco/alfresco-transform-core). + +## Contributing guide + +Please use [this guide](https://github.com/Alfresco/alfresco-repository/blob/master/CONTRIBUTING.md) +to make a contribution to the project. + diff --git a/t-engine-base/pom.xml b/t-engine-base/pom.xml new file mode 100644 index 00000000..95fdb002 --- /dev/null +++ b/t-engine-base/pom.xml @@ -0,0 +1,101 @@ + + + 4.0.0 + + + org.alfresco + alfresco-transform-core + 2.6.1-SNAPSHOT + + + alfresco-t-engine-base + + + false + + + + + org.springframework.boot + spring-boot-starter-thymeleaf + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-actuator + + + io.micrometer + micrometer-registry-prometheus + + + org.springframework.boot + spring-boot-starter-test + test + + + com.vaadin.external.google + android-json + + + + + + org.dom4j + dom4j + + + + org.alfresco + alfresco-transform-model + ${project.version} + + + org.springframework.boot + spring-boot-starter-activemq + + + org.apache.activemq + activemq-client + + + com.fasterxml.jackson.core + jackson-annotations + + + org.messaginghub + pooled-jms + + + com.google.collections + google-collections + 1.0 + + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + ${transformer.base.deploy.skip} + + + + + diff --git a/alfresco-transform-tika/alfresco-transform-tika-boot/src/main/java/org/alfresco/transformer/Application.java b/t-engine-base/src/main/java/org/alfresco/transform/base/Application.java similarity index 79% rename from alfresco-transform-tika/alfresco-transform-tika-boot/src/main/java/org/alfresco/transformer/Application.java rename to t-engine-base/src/main/java/org/alfresco/transform/base/Application.java index d95be427..582da7d4 100644 --- a/alfresco-transform-tika/alfresco-transform-tika-boot/src/main/java/org/alfresco/transformer/Application.java +++ b/t-engine-base/src/main/java/org/alfresco/transform/base/Application.java @@ -2,7 +2,7 @@ * #%L * Alfresco Transform Core * %% - * Copyright (C) 2005 - 2020 Alfresco Software Limited + * Copyright (C) 2005 - 2022 Alfresco Software Limited * %% * This file is part of the Alfresco software. * - @@ -24,12 +24,12 @@ * along with Alfresco. If not, see . * #L% */ -package org.alfresco.transformer; +package org.alfresco.transform.base; import io.micrometer.core.instrument.MeterRegistry; -import org.alfresco.transformer.executors.TikaJavaExecutor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.SpringApplication; import org.springframework.boot.actuate.autoconfigure.metrics.MeterRegistryCustomizer; @@ -41,15 +41,19 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.event.EventListener; import java.util.Arrays; +import java.util.List; -import static org.alfresco.transformer.logging.StandardMessages.LICENCE; +import static org.alfresco.transform.base.logging.StandardMessages.LICENCE; @SpringBootApplication -@EnableAutoConfiguration(exclude = {DataSourceAutoConfiguration.class}) +@EnableAutoConfiguration(exclude = { DataSourceAutoConfiguration.class}) public class Application { private static final Logger logger = LoggerFactory.getLogger(Application.class); + @Autowired(required = false) + private List transformEngines; + @Value("${container.name}") private String containerName; @@ -69,7 +73,11 @@ public class Application { logger.info("--------------------------------------------------------------------------------------------------------------------------------------------------------------"); Arrays.stream(LICENCE.split("\\n")).forEach(logger::info); - Arrays.stream(TikaJavaExecutor.LICENCE.split("\\n")).forEach(logger::info); + if (transformEngines != null) { + transformEngines.stream() + .map(transformEngine -> transformEngine.getStartupMessage()) + .forEach(message -> Arrays.stream(message.split("\\n")).forEach(logger::info)); + } logger.info("--------------------------------------------------------------------------------------------------------------------------------------------------------------"); logger.info("Starting application components... Done"); diff --git a/t-engine-base/src/main/java/org/alfresco/transform/base/CustomTransformer.java b/t-engine-base/src/main/java/org/alfresco/transform/base/CustomTransformer.java new file mode 100644 index 00000000..f36948cd --- /dev/null +++ b/t-engine-base/src/main/java/org/alfresco/transform/base/CustomTransformer.java @@ -0,0 +1,40 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.base; + +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Map; + +public interface CustomTransformer +{ + String getTransformerName(); + + void transform(String sourceMimetype, String sourceEncoding, InputStream inputStream, + String targetMimetype, String targetEncoding, OutputStream outputStream, + Map transformOptions) throws Exception; +} diff --git a/t-engine-base/src/main/java/org/alfresco/transform/base/QueueTransformService.java b/t-engine-base/src/main/java/org/alfresco/transform/base/QueueTransformService.java new file mode 100644 index 00000000..fb500916 --- /dev/null +++ b/t-engine-base/src/main/java/org/alfresco/transform/base/QueueTransformService.java @@ -0,0 +1,208 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.base; + +import static org.springframework.http.HttpStatus.BAD_REQUEST; +import static org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR; + +import java.util.Optional; + +import javax.jms.Destination; +import javax.jms.JMSException; +import javax.jms.Message; + +import org.alfresco.transform.client.model.TransformReply; +import org.alfresco.transform.client.model.TransformRequest; +import org.alfresco.transform.common.TransformException; +import org.alfresco.transform.base.messaging.TransformMessageConverter; +import org.alfresco.transform.base.messaging.TransformReplySender; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.http.HttpStatus; +import org.springframework.jms.annotation.JmsListener; +import org.springframework.jms.support.converter.MessageConversionException; +import org.springframework.stereotype.Component; + +/** + * Queue Transformer service. + * This service reads all the requests for the particular engine, forwards them to the worker + * component (at this time the injected controller - to be refactored) and sends back the reply + * to the {@link Message#getJMSReplyTo()} value. If this value is missing we've got to a dead end. + * + * @author Lucian Tuca + * created on 18/12/2018 + */ +@Component +@ConditionalOnProperty(name = "activemq.url") +public class QueueTransformService +{ + private static final Logger logger = LoggerFactory.getLogger(QueueTransformService.class); + + // TODO: I know this is not smart but all the the transformation logic is in the Controller. + // The controller also manages the probes. There's tons of refactoring needed there, hence this. Sorry. + @Autowired + private TransformController transformController; + + @Autowired + private TransformMessageConverter transformMessageConverter; + + @Autowired + private TransformReplySender transformReplySender; + + @JmsListener(destination = "${queue.engineRequestQueue}", concurrency = "${jms-listener.concurrency}") + public void receive(final Message msg) + { + if (msg == null) + { + logger.error("Received null message!"); + return; + } + + final String correlationId = tryRetrieveCorrelationId(msg); + Destination replyToDestinationQueue; + + try + { + replyToDestinationQueue = msg.getJMSReplyTo(); + if (replyToDestinationQueue == null) + { + logger.error( + "Cannot find 'replyTo' destination queue for message with correlationID {}. Stopping. ", + correlationId); + return; + } + } + catch (JMSException e) + { + logger.error( + "Cannot find 'replyTo' destination queue for message with correlationID {}. Stopping. ", + correlationId); + return; + } + + logger.trace("New T-Request from queue with correlationId: {}", correlationId); + + Optional transformRequest; + try + { + transformRequest = convert(msg, correlationId); + } + catch (TransformException e) + { + logger.error(e.getMessage(), e); + replyWithError(replyToDestinationQueue, HttpStatus.valueOf(e.getStatusCode()), + e.getMessage(), correlationId); + return; + } + + if (!transformRequest.isPresent()) + { + logger.error("T-Request from message with correlationID {} is null!", correlationId); + replyWithInternalSvErr(replyToDestinationQueue, + "JMS exception during T-Request deserialization: ", correlationId); + return; + } + + TransformReply reply = transformController.transform(transformRequest.get(), null) + .getBody(); + + transformReplySender.send(replyToDestinationQueue, reply); + } + + /** + * Tries to convert the JMS {@link Message} to a {@link TransformRequest} + * If any error occurs, a {@link TransformException} is thrown + * + * @param msg Message to be deserialized + * @return The converted {@link TransformRequest} instance + */ + private Optional convert(final Message msg, String correlationId) + { + try + { + TransformRequest request = (TransformRequest) transformMessageConverter + .fromMessage(msg); + return Optional.ofNullable(request); + } + catch (MessageConversionException e) + { + String message = + "MessageConversionException during T-Request deserialization of message with correlationID " + + correlationId + ": "; + throw new TransformException(BAD_REQUEST.value(), message + e.getMessage()); + } + catch (JMSException e) + { + String message = + "JMSException during T-Request deserialization of message with correlationID " + + correlationId + ": "; + throw new TransformException(INTERNAL_SERVER_ERROR.value(), + message + e.getMessage()); + } + catch (Exception e) + { + String message = + "Exception during T-Request deserialization of message with correlationID " + + correlationId + ": "; + throw new TransformException(INTERNAL_SERVER_ERROR.value(), + message + e.getMessage()); + } + } + + private void replyWithInternalSvErr(final Destination destination, final String msg, + final String correlationId) + { + replyWithError(destination, INTERNAL_SERVER_ERROR, msg, correlationId); + } + + private void replyWithError(final Destination destination, final HttpStatus status, + final String msg, + final String correlationId) + { + final TransformReply reply = TransformReply + .builder() + .withStatus(status.value()) + .withErrorDetails(msg) + .build(); + + transformReplySender.send(destination, reply, correlationId); + } + + private static String tryRetrieveCorrelationId(final Message msg) + { + try + { + return msg.getJMSCorrelationID(); + } + catch (Exception ignore) + { + return null; + } + } +} diff --git a/t-engine-base/src/main/java/org/alfresco/transform/base/TransformController.java b/t-engine-base/src/main/java/org/alfresco/transform/base/TransformController.java new file mode 100644 index 00000000..2c5a9a54 --- /dev/null +++ b/t-engine-base/src/main/java/org/alfresco/transform/base/TransformController.java @@ -0,0 +1,662 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.base; + +import org.alfresco.transform.base.probes.ProbeTestTransform; +import org.alfresco.transform.common.TransformerDebug; +import org.alfresco.transform.client.model.InternalContext; +import org.alfresco.transform.client.model.TransformReply; +import org.alfresco.transform.client.model.TransformRequest; +import org.alfresco.transform.messages.TransformRequestValidator; +import org.alfresco.transform.config.TransformConfig; +import org.alfresco.transform.registry.TransformServiceRegistry; +import org.alfresco.transform.common.TransformException; +import org.alfresco.transform.base.clients.AlfrescoSharedFileStoreClient; +import org.alfresco.transform.base.logging.LogEntry; +import org.alfresco.transform.base.model.FileRefResponse; +import org.codehaus.plexus.util.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.TypeMismatchException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.core.io.Resource; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Controller; +import org.springframework.ui.Model; +import org.springframework.validation.DirectFieldBindingResult; +import org.springframework.validation.Errors; +import org.springframework.web.bind.MissingServletRequestParameterException; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.multipart.MultipartFile; + +import javax.annotation.PostConstruct; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +import static java.text.MessageFormat.format; +import static java.util.stream.Collectors.joining; +import static org.alfresco.transform.config.CoreVersionDecorator.setOrClearCoreVersion; +import static org.alfresco.transform.common.RequestParamMap.DIRECT_ACCESS_URL; +import static org.alfresco.transform.common.RequestParamMap.CONFIG_VERSION; +import static org.alfresco.transform.common.RequestParamMap.CONFIG_VERSION_DEFAULT; +import static org.alfresco.transform.common.RequestParamMap.ENDPOINT_TRANSFORM; +import static org.alfresco.transform.common.RequestParamMap.ENDPOINT_TRANSFORM_CONFIG; +import static org.alfresco.transform.base.fs.FileManager.TempFileProvider.createTempFile; +import static org.alfresco.transform.base.fs.FileManager.buildFile; +import static org.alfresco.transform.base.fs.FileManager.createAttachment; +import static org.alfresco.transform.base.fs.FileManager.createSourceFile; +import static org.alfresco.transform.base.fs.FileManager.createTargetFile; +import static org.alfresco.transform.base.fs.FileManager.createTargetFileName; +import static org.alfresco.transform.base.fs.FileManager.deleteFile; +import static org.alfresco.transform.base.fs.FileManager.getFilenameFromContentDisposition; +import static org.alfresco.transform.base.fs.FileManager.save; +import static org.alfresco.transform.base.util.RequestParamMap.FILE; +import static org.alfresco.transform.base.util.RequestParamMap.SOURCE_ENCODING; +import static org.alfresco.transform.base.util.RequestParamMap.SOURCE_EXTENSION; +import static org.alfresco.transform.base.util.RequestParamMap.SOURCE_MIMETYPE; +import static org.alfresco.transform.base.util.RequestParamMap.TARGET_MIMETYPE; +import static org.springframework.http.HttpStatus.BAD_REQUEST; +import static org.springframework.http.HttpStatus.CREATED; +import static org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR; +import static org.springframework.http.HttpStatus.OK; +import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; +import static org.springframework.http.MediaType.MULTIPART_FORM_DATA_VALUE; +import static org.springframework.util.StringUtils.getFilenameExtension; + +/** + * Provides the main endpoints into the t-engine. + */ +@Controller +public class TransformController +{ + private static final Logger logger = LoggerFactory.getLogger(TransformController.class); + private static final List NON_TRANSFORM_OPTION_REQUEST_PARAMETERS = Arrays.asList(SOURCE_EXTENSION, + TARGET_MIMETYPE, SOURCE_MIMETYPE, DIRECT_ACCESS_URL); + + @Autowired(required = false) + private List transformEngines; + + @Autowired(required = false) + private List customTransformers; + + @Autowired + private AlfrescoSharedFileStoreClient alfrescoSharedFileStoreClient; + @Autowired + private TransformRequestValidator transformRequestValidator; + @Autowired + private TransformServiceRegistry transformRegistry; + @Autowired + private TransformerDebug transformerDebug; + @Value("${transform.core.version}") + private String coreVersion; + + private TransformEngine transformEngine; + ProbeTestTransform probeTestTransform; + private Map customTransformersByName = new HashMap<>(); + private AtomicInteger httpRequestCount = new AtomicInteger(1); + + @PostConstruct + public void init() + { + initTransformEngine(); + initProbeTestTransform(); + initCustomTransformersByName(); + } + + private void initTransformEngine() + { + if (transformEngines != null) + { + // Normally there is just one TransformEngine per t-engine, but we also want to be able to amalgamate the + // CustomTransform code from many t-engines into a single t-engine. In this case, there should be a wrapper + // TransformEngine (it has no TransformConfig of its own). + transformEngine = transformEngines.stream() + .filter(transformEngine -> transformEngine.getTransformConfig() == null) + .findFirst() + .orElse(transformEngines.get(0)); + } + } + + private void initProbeTestTransform() + { + if (transformEngine != null) + { + probeTestTransform = transformEngine.getLivenessAndReadinessProbeTestTransform(); + } + } + + private void initCustomTransformersByName() + { + if (customTransformers != null) + { + customTransformers.forEach(customTransformer -> customTransformersByName.put(customTransformer.getTransformerName(), + customTransformer)); + } + } + + /** + * @return a string that may be used in client debug. + */ + @RequestMapping("/version") + @ResponseBody + public String version() + { + return transformEngine.getTransformEngineName() + ' ' + coreVersion + " available"; + } + + /** + * Test UI page to perform a transform. + */ + @GetMapping("/") + public String transformForm(Model model) + { + return "transformForm"; + } + + /** + * Test UI error page. + */ + @GetMapping("/error") + public String error() + { + return "error"; // the name of the template + } + + /** + * Test UI log page. + */ + @GetMapping("/log") + String log(Model model) + { + model.addAttribute("title", transformEngine.getTransformEngineName() + " Log Entries"); + Collection log = LogEntry.getLog(); + if (!log.isEmpty()) + { + model.addAttribute("log", log); + } + return "log"; // the name of the template + } + + /** + * Kubernetes readiness probe. + */ + @GetMapping("/ready") + @ResponseBody + public String ready(HttpServletRequest request) + { + return probeTestTransform.doTransformOrNothing(request, false, this); + } + + /** + * Kubernetes liveness probe. + */ + @GetMapping("/live") + @ResponseBody + public String live(HttpServletRequest request) + { + return probeTestTransform.doTransformOrNothing(request, true, this); + } + + @GetMapping(value = ENDPOINT_TRANSFORM_CONFIG) + public ResponseEntity transformConfig( + @RequestParam(value = CONFIG_VERSION, defaultValue = CONFIG_VERSION_DEFAULT) int configVersion) + { + logger.info("GET Transform Config version: " + configVersion); + TransformConfig transformConfig = ((TransformRegistryImpl) transformRegistry).getTransformConfig(); + transformConfig = setOrClearCoreVersion(transformConfig, configVersion); + return new ResponseEntity<>(transformConfig, OK); + } + + @PostMapping(value = ENDPOINT_TRANSFORM, consumes = MULTIPART_FORM_DATA_VALUE) + public ResponseEntity transform(HttpServletRequest request, + @RequestParam(value = FILE, required = false) MultipartFile sourceMultipartFile, + @RequestParam(value = SOURCE_MIMETYPE, required = false) String sourceMimetype, + @RequestParam(value = TARGET_MIMETYPE, required = false) String targetMimetype, + @RequestParam Map requestParameters) + { + if (logger.isDebugEnabled()) + { + logger.debug("Processing request via HTTP endpoint. Params: sourceMimetype: '{}', targetMimetype: '{}', " + + "requestParameters: {}", sourceMimetype, targetMimetype, requestParameters); + } + + final String directUrl = requestParameters.getOrDefault(DIRECT_ACCESS_URL, ""); + + File sourceFile; + String sourceFilename; + if (directUrl.isBlank()) + { + if (sourceMultipartFile == null) + { + throw new TransformException(BAD_REQUEST.value(), "Required request part 'file' is not present"); + } + sourceFile = createSourceFile(request, sourceMultipartFile); + sourceFilename = sourceMultipartFile.getOriginalFilename(); + } + else + { + sourceFile = getSourceFileFromDirectUrl(directUrl); + sourceFilename = sourceFile.getName(); + } + + final String targetFilename = createTargetFileName(sourceFilename, sourceMimetype, targetMimetype); + probeTestTransform.incrementTransformerCount(); + final File targetFile = createTargetFile(request, targetFilename); + + Map transformOptions = getTransformOptions(requestParameters); + String transformName = getTransformerName(sourceFile, sourceMimetype, targetMimetype, transformOptions); + String reference = "e"+httpRequestCount.getAndIncrement(); + transformerDebug.pushTransform(reference, sourceMimetype, targetMimetype, sourceFile, transformName); + transformerDebug.logOptions(reference, requestParameters); + try + { + transformImpl(transformName, sourceMimetype, targetMimetype, transformOptions, sourceFile, targetFile); + + final ResponseEntity body = createAttachment(targetFilename, targetFile); + LogEntry.setTargetSize(targetFile.length()); + long time = LogEntry.setStatusCodeAndMessage(OK.value(), "Success"); + probeTestTransform.recordTransformTime(time); + transformerDebug.popTransform(reference, time); + return body; + } + catch (Throwable t) + { + transformerDebug.logFailure(reference, t.getMessage()); + throw t; + } + } + + /** + * '/transform' endpoint which consumes and produces 'application/json' + * + * This is the way to tell Spring to redirect the request to this endpoint + * instead of the one which produces 'html' + * + * @param request The transformation request + * @param timeout Transformation timeout + * @return A transformation reply + */ + @PostMapping(value = ENDPOINT_TRANSFORM, produces = APPLICATION_JSON_VALUE) + @ResponseBody + public ResponseEntity transform(@RequestBody TransformRequest request, + @RequestParam(value = "timeout", required = false) Long timeout) + { + logger.trace("Received {}, timeout {} ms", request, timeout); + + final TransformReply reply = new TransformReply(); + reply.setRequestId(request.getRequestId()); + reply.setSourceReference(request.getSourceReference()); + reply.setSchema(request.getSchema()); + reply.setClientData(request.getClientData()); + + final Errors errors = validateTransformRequest(request); + validateInternalContext(request, errors); + initialiseContext(request); + reply.setInternalContext(request.getInternalContext()); + if (!errors.getAllErrors().isEmpty()) + { + reply.setStatus(BAD_REQUEST.value()); + reply.setErrorDetails(errors + .getAllErrors() + .stream() + .map(Object::toString) + .collect(joining(", "))); + + transformerDebug.logFailure(reply); + logger.trace("Invalid request, sending {}", reply); + return new ResponseEntity<>(reply, HttpStatus.valueOf(reply.getStatus())); + } + transformerDebug.pushTransform(request); + + // Load the source file + File sourceFile; + try + { + final String directUrl = request.getTransformRequestOptions().getOrDefault(DIRECT_ACCESS_URL, ""); + if (directUrl.isBlank()) + { + sourceFile = loadSourceFile(request.getSourceReference(), request.getSourceExtension()); + } + else + { + sourceFile = getSourceFileFromDirectUrl(directUrl); + } + } + catch (TransformException e) + { + reply.setStatus(e.getStatusCode()); + reply.setErrorDetails(messageWithCause("Failed at reading the source file", e)); + + transformerDebug.logFailure(reply); + logger.trace("Failed to load source file (TransformException), sending " + reply); + return new ResponseEntity<>(reply, HttpStatus.valueOf(reply.getStatus())); + } + catch (HttpClientErrorException e) + { + reply.setStatus(e.getStatusCode().value()); + reply.setErrorDetails(messageWithCause("Failed at reading the source file", e)); + + transformerDebug.logFailure(reply); + logger.trace("Failed to load source file (HttpClientErrorException), sending " + reply, e); + return new ResponseEntity<>(reply, HttpStatus.valueOf(reply.getStatus())); + } + catch (Exception e) + { + reply.setStatus(INTERNAL_SERVER_ERROR.value()); + reply.setErrorDetails(messageWithCause("Failed at reading the source file", e)); + + transformerDebug.logFailure(reply); + logger.trace("Failed to load source file (Exception), sending " + reply, e); + return new ResponseEntity<>(reply, HttpStatus.valueOf(reply.getStatus())); + } + + // Create local temp target file in order to run the transformation + final String targetFilename = createTargetFileName(sourceFile.getName(), request.getTargetMediaType(), request.getSourceMediaType()); + final File targetFile = buildFile(targetFilename); + + // Run the transformation + try + { + String targetMimetype = request.getTargetMediaType(); + String sourceMimetype = request.getSourceMediaType(); + Map transformOptions = getTransformOptions(request.getTransformRequestOptions()); + transformerDebug.logOptions(request); + String transformName = getTransformerName(sourceFile, sourceMimetype, targetMimetype, transformOptions); + transformImpl(transformName, sourceMimetype, targetMimetype, transformOptions, sourceFile, targetFile); + reply.getInternalContext().setCurrentSourceSize(targetFile.length()); + } + catch (TransformException e) + { + reply.setStatus(e.getStatusCode()); + reply.setErrorDetails(messageWithCause("Failed at processing transformation", e)); + + transformerDebug.logFailure(reply); + logger.trace("Failed to perform transform (TransformException), sending " + reply, e); + return new ResponseEntity<>(reply, HttpStatus.valueOf(reply.getStatus())); + } + catch (Exception e) + { + reply.setStatus(INTERNAL_SERVER_ERROR.value()); + reply.setErrorDetails(messageWithCause("Failed at processing transformation", e)); + + transformerDebug.logFailure(reply); + logger.trace("Failed to perform transform (Exception), sending " + reply, e); + return new ResponseEntity<>(reply, HttpStatus.valueOf(reply.getStatus())); + } + + // Write the target file + FileRefResponse targetRef; + try + { + targetRef = alfrescoSharedFileStoreClient.saveFile(targetFile); + } + catch (TransformException e) + { + reply.setStatus(e.getStatusCode()); + reply.setErrorDetails(messageWithCause("Failed at writing the transformed file", e)); + + transformerDebug.logFailure(reply); + logger.trace("Failed to save target file (TransformException), sending " + reply, e); + return new ResponseEntity<>(reply, HttpStatus.valueOf(reply.getStatus())); + } + catch (HttpClientErrorException e) + { + reply.setStatus(e.getStatusCode().value()); + reply.setErrorDetails(messageWithCause("Failed at writing the transformed file. ", e)); + + transformerDebug.logFailure(reply); + logger.trace("Failed to save target file (HttpClientErrorException), sending " + reply, e); + return new ResponseEntity<>(reply, HttpStatus.valueOf(reply.getStatus())); + } + catch (Exception e) + { + reply.setStatus(INTERNAL_SERVER_ERROR.value()); + reply.setErrorDetails(messageWithCause("Failed at writing the transformed file. ", e)); + + transformerDebug.logFailure(reply); + logger.trace("Failed to save target file (Exception), sending " + reply, e); + return new ResponseEntity<>(reply, HttpStatus.valueOf(reply.getStatus())); + } + + try + { + deleteFile(targetFile); + } + catch (Exception e) + { + logger.error("Failed to delete local temp target file '{}'. Error will be ignored ", + targetFile, e); + } + try + { + deleteFile(sourceFile); + } + catch (Exception e) + { + logger.error("Failed to delete source local temp file " + sourceFile, e); + } + + reply.setTargetReference(targetRef.getEntry().getFileRef()); + reply.setStatus(CREATED.value()); + + transformerDebug.popTransform(reply); + logger.trace("Sending successful {}, timeout {} ms", reply, timeout); + return new ResponseEntity<>(reply, HttpStatus.valueOf(reply.getStatus())); + } + + private Errors validateTransformRequest(final TransformRequest transformRequest) + { + DirectFieldBindingResult errors = new DirectFieldBindingResult(transformRequest, "request"); + transformRequestValidator.validate(transformRequest, errors); + return errors; + } + + private void validateInternalContext(TransformRequest request, Errors errors) + { + String errorMessage = InternalContext.checkForBasicErrors(request.getInternalContext(), "T-Request"); + if (errorMessage != null) + { + errors.rejectValue("internalContext", null, errorMessage); + } + } + + private void initialiseContext(TransformRequest request) + { + // If needed, initialise the context enough to allow logging to take place without NPE checks + request.setInternalContext(InternalContext.initialise(request.getInternalContext())); + } + + private File getSourceFileFromDirectUrl(String directUrl) + { + File sourceFile = createTempFile("tmp", ".tmp"); + try + { + FileUtils.copyURLToFile(new URL(directUrl), sourceFile); + } + catch (IllegalArgumentException e) + { + throw new TransformException(BAD_REQUEST.value(), "Direct Access Url is invalid.", e); + } + catch (IOException e) + { + throw new TransformException(BAD_REQUEST.value(), "Direct Access Url not found.", e); + } + + return sourceFile; + } + + protected Map getTransformOptions(Map requestParameters) + { + Map transformOptions = new HashMap<>(requestParameters); + transformOptions.keySet().removeAll(NON_TRANSFORM_OPTION_REQUEST_PARAMETERS); + transformOptions.values().removeIf(v -> v.isEmpty()); + return transformOptions; + } + + /** + * Loads the file with the specified sourceReference from Alfresco Shared File Store + * + * @param sourceReference reference to the file in Alfresco Shared File Store + * @param sourceExtension default extension if the file in Alfresco Shared File Store has none + * @return the file containing the source content for the transformation + */ + private File loadSourceFile(final String sourceReference, final String sourceExtension) + { + ResponseEntity responseEntity = alfrescoSharedFileStoreClient + .retrieveFile(sourceReference); + probeTestTransform.incrementTransformerCount(); + + HttpHeaders headers = responseEntity.getHeaders(); + String filename = getFilenameFromContentDisposition(headers); + + String extension = getFilenameExtension(filename) != null ? getFilenameExtension(filename) : sourceExtension; + MediaType contentType = headers.getContentType(); + long size = headers.getContentLength(); + + final Resource body = responseEntity.getBody(); + if (body == null) + { + String message = "Source file with reference: " + sourceReference + " is null or empty. " + + "Transformation will fail and stop now as there is no content to be transformed."; + logger.warn(message); + throw new TransformException(BAD_REQUEST.value(), message); + } + final File file = createTempFile("source_", "." + extension); + + logger.debug("Read source content {} length={} contentType={}", + sourceReference, size, contentType); + + save(body, file); + LogEntry.setSource(filename, size); + return file; + } + + private static String messageWithCause(final String prefix, Throwable e) + { + final StringBuilder sb = new StringBuilder(); + sb.append(prefix).append(" - ") + .append(e.getClass().getSimpleName()).append(": ") + .append(e.getMessage()); + + while (e.getCause() != null) + { + e = e.getCause(); + sb.append(", cause ") + .append(e.getClass().getSimpleName()).append(": ") + .append(e.getMessage()); + } + + return sb.toString(); + } + + private String getTransformerName(final File sourceFile, final String sourceMimetype, + final String targetMimetype, final Map transformOptions) + { + // The transformOptions always contains sourceEncoding when sent to a T-Engine, even though it should not be + // used to select a transformer. Similar to source and target mimetypes and extensions, but these are not + // passed in transformOptions. + String sourceEncoding = transformOptions.remove(SOURCE_ENCODING); + try + { + final long sourceSizeInBytes = sourceFile.length(); + final String transformerName = transformRegistry.findTransformerName(sourceMimetype, + sourceSizeInBytes, targetMimetype, transformOptions, null); + if (transformerName == null) + { + throw new TransformException(BAD_REQUEST.value(), "No transforms were able to handle the request"); + } + return transformerName; + } + finally + { + if (sourceEncoding != null) + { + transformOptions.put(SOURCE_ENCODING, sourceEncoding); + } + } + } + + public void transformImpl(String transformName, String sourceMimetype, String targetMimetype, + Map transformOptions, File sourceFile, File targetFile) + { + //javaExecutor.transformExtractOrEmbed(transformName, sourceMimetype, targetMimetype, transformOptions, sourceFile, targetFile); + } + + @ExceptionHandler(TypeMismatchException.class) + public void handleParamsTypeMismatch(HttpServletResponse response, MissingServletRequestParameterException e) throws IOException + { + final String message = format("Request parameter ''{0}'' is of the wrong type", e.getParameterName()); + final int statusCode = BAD_REQUEST.value(); + + logger.error(message, e); + LogEntry.setStatusCodeAndMessage(statusCode, message); + response.sendError(statusCode, transformEngine.getTransformEngineName() + " - " + message); + } + + @ExceptionHandler(MissingServletRequestParameterException.class) + public void handleMissingParams(HttpServletResponse response, MissingServletRequestParameterException e) throws IOException + { + final String message = format("Request parameter ''{0}'' is missing", e.getParameterName()); + final int statusCode = BAD_REQUEST.value(); + + logger.error(message, e); + LogEntry.setStatusCodeAndMessage(statusCode, message); + response.sendError(statusCode, transformEngine.getTransformEngineName() + " - " + message); + } + + @ExceptionHandler(TransformException.class) + public void transformExceptionWithMessage(HttpServletResponse response, TransformException e) throws IOException + { + final String message = e.getMessage(); + final int statusCode = e.getStatusCode(); + + logger.error(message, e); + long time = LogEntry.setStatusCodeAndMessage(statusCode, message); + probeTestTransform.recordTransformTime(time); + response.sendError(statusCode, transformEngine.getTransformEngineName() + " - " + message); + } +} diff --git a/t-engine-base/src/main/java/org/alfresco/transform/base/TransformEngine.java b/t-engine-base/src/main/java/org/alfresco/transform/base/TransformEngine.java new file mode 100644 index 00000000..527cb449 --- /dev/null +++ b/t-engine-base/src/main/java/org/alfresco/transform/base/TransformEngine.java @@ -0,0 +1,56 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.base; + +import org.alfresco.transform.config.TransformConfig; +import org.alfresco.transform.base.probes.ProbeTestTransform; + +/** + * The interface to the custom transform code applied on top of a base t-engine. + */ +public interface TransformEngine +{ + /** + * @return the name of the t-engine. The t-router reads config from t-engines in name order. + */ + String getTransformEngineName(); + + /** + * @return messages to be logged on start up (license & settings). Use \n to split onto multiple lines. + */ + String getStartupMessage(); + + /** + * @return a definition of what the t-engine supports. Normally read from a json Resource on the classpath. + */ + TransformConfig getTransformConfig(); + + /** + * @return a ProbeTestTransform (will do a quick transform) for k8 liveness and readiness probes. + */ + ProbeTestTransform getLivenessAndReadinessProbeTestTransform(); +} diff --git a/t-engine-base/src/main/java/org/alfresco/transform/base/TransformInterceptor.java b/t-engine-base/src/main/java/org/alfresco/transform/base/TransformInterceptor.java new file mode 100644 index 00000000..16f5d3f2 --- /dev/null +++ b/t-engine-base/src/main/java/org/alfresco/transform/base/TransformInterceptor.java @@ -0,0 +1,64 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.base; + +import static org.alfresco.transform.base.fs.FileManager.SOURCE_FILE; +import static org.alfresco.transform.base.fs.FileManager.TARGET_FILE; +import static org.alfresco.transform.base.fs.FileManager.deleteFile; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.alfresco.transform.base.logging.LogEntry; +import org.springframework.web.servlet.handler.HandlerInterceptorAdapter; + +/** + * TransformInterceptor + *
+ * Handles ThreadLocal Log entries for each request. + */ +public class TransformInterceptor extends HandlerInterceptorAdapter +{ + @Override + public boolean preHandle(HttpServletRequest request, + HttpServletResponse response, Object handler) + { + LogEntry.start(); + return true; + } + + @Override + public void afterCompletion(HttpServletRequest request, + HttpServletResponse response, Object handler, Exception ex) + { + // TargetFile cannot be deleted until completion, otherwise 0 bytes are sent. + deleteFile(request, SOURCE_FILE); + deleteFile(request, TARGET_FILE); + + LogEntry.complete(); + } +} diff --git a/t-engine-base/src/main/java/org/alfresco/transform/base/TransformRegistryImpl.java b/t-engine-base/src/main/java/org/alfresco/transform/base/TransformRegistryImpl.java new file mode 100644 index 00000000..c4ae7812 --- /dev/null +++ b/t-engine-base/src/main/java/org/alfresco/transform/base/TransformRegistryImpl.java @@ -0,0 +1,106 @@ +/* + * #%L + * Alfresco Repository + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.base; + +import org.alfresco.transform.config.TransformConfig; +import org.alfresco.transform.registry.AbstractTransformRegistry; +import org.alfresco.transform.registry.CombinedTransformConfig; +import org.alfresco.transform.registry.TransformCache; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; + +import javax.annotation.PostConstruct; +import java.util.Comparator; +import java.util.List; + +import static org.alfresco.transform.config.CoreVersionDecorator.setCoreVersionOnSingleStepTransformers; + +/** + * Used by clients to work out if a transformation is supported based on the engine_config.json. + */ +public class TransformRegistryImpl extends AbstractTransformRegistry +{ + private static final Logger log = LoggerFactory.getLogger(TransformRegistryImpl.class); + + @Autowired(required = false) + private List transformEngines; + + @Value("${transform.core.version}") + private String coreVersion; + + private TransformConfig transformConfigBeforeIncompleteTransformsAreRemoved; + + @PostConstruct + public void init() + { + CombinedTransformConfig combinedTransformConfig = new CombinedTransformConfig(); + if (transformEngines != null) + { + transformEngines.stream() + .sorted(Comparator.comparing(TransformEngine::getTransformEngineName)) + .forEach(transformEngine -> { + TransformConfig transformConfig = transformEngine.getTransformConfig(); + if (transformConfig != null) + { + setCoreVersionOnSingleStepTransformers(transformConfig, coreVersion); + combinedTransformConfig.addTransformConfig(transformConfig, + transformEngine.getTransformEngineName(), "---", this); + } + }); + } + transformConfigBeforeIncompleteTransformsAreRemoved = combinedTransformConfig.buildTransformConfig(); + combinedTransformConfig.combineTransformerConfig(this); + combinedTransformConfig.registerCombinedTransformers(this); + } + + // Unlike other subclasses this class does not extend Data or replace it at run time. + private TransformCache data = new TransformCache(); + + public TransformConfig getTransformConfig() + { + return transformConfigBeforeIncompleteTransformsAreRemoved; + } + + @Override + public TransformCache getData() + { + return data; + } + + @Override + protected void logError(String msg) + { + log.error(msg); + } + + @Override + protected void logWarn(String msg) + { + log.warn(msg); + } +} diff --git a/t-engine-base/src/main/java/org/alfresco/transform/base/clients/AlfrescoSharedFileStoreClient.java b/t-engine-base/src/main/java/org/alfresco/transform/base/clients/AlfrescoSharedFileStoreClient.java new file mode 100644 index 00000000..bd608712 --- /dev/null +++ b/t-engine-base/src/main/java/org/alfresco/transform/base/clients/AlfrescoSharedFileStoreClient.java @@ -0,0 +1,103 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.base.clients; + +import static org.springframework.http.HttpMethod.POST; +import static org.springframework.http.MediaType.MULTIPART_FORM_DATA; + +import java.io.File; + +import org.alfresco.transform.common.TransformException; +import org.alfresco.transform.base.model.FileRefResponse; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.core.io.FileSystemResource; +import org.springframework.core.io.Resource; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.ResponseEntity; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.RestTemplate; + +/** + * Simple Rest client that call Alfresco Shared File Store + */ +public class AlfrescoSharedFileStoreClient +{ + @Value("${fileStoreUrl}") + private String fileStoreUrl; + + @Autowired + private RestTemplate restTemplate; + + /** + * Retrieves a file from Shared File Store using given file reference + * + * @param fileRef File reference + * @return ResponseEntity + */ + public ResponseEntity retrieveFile(String fileRef) + { + try + { + return restTemplate.getForEntity(fileStoreUrl + "/" + fileRef, + org.springframework.core.io.Resource.class); + } + catch (HttpClientErrorException e) + { + throw new TransformException(e.getStatusCode().value(), e.getMessage(), e); + } + } + + /** + * Stores given file in Shared File Store + * + * @param file File to be stored + * @return A FileRefResponse containing detail about file's reference + */ + public FileRefResponse saveFile(File file) + { + try + { + FileSystemResource value = new FileSystemResource(file.getAbsolutePath()); + LinkedMultiValueMap map = new LinkedMultiValueMap<>(); + map.add("file", value); + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MULTIPART_FORM_DATA); + HttpEntity> requestEntity = new HttpEntity<>(map, + headers); + ResponseEntity responseEntity = restTemplate + .exchange(fileStoreUrl, POST, requestEntity, FileRefResponse.class); + return responseEntity.getBody(); + } + catch (HttpClientErrorException e) + { + throw new TransformException(e.getStatusCode().value(), e.getMessage(), e); + } + } +} diff --git a/t-engine-base/src/main/java/org/alfresco/transform/base/config/WebApplicationConfig.java b/t-engine-base/src/main/java/org/alfresco/transform/base/config/WebApplicationConfig.java new file mode 100644 index 00000000..b122d692 --- /dev/null +++ b/t-engine-base/src/main/java/org/alfresco/transform/base/config/WebApplicationConfig.java @@ -0,0 +1,95 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.base.config; + +import org.alfresco.transform.base.TransformInterceptor; +import org.alfresco.transform.base.TransformRegistryImpl; +import org.alfresco.transform.base.clients.AlfrescoSharedFileStoreClient; +import org.alfresco.transform.common.TransformerDebug; +import org.alfresco.transform.messages.TransformRequestValidator; +import org.alfresco.transform.registry.TransformServiceRegistry; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.env.Environment; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.servlet.config.annotation.InterceptorRegistry; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; + +import static org.alfresco.transform.common.RequestParamMap.ENDPOINT_TRANSFORM; + +@Configuration +@ComponentScan(basePackages = {"org.alfresco.transform"}) +public class WebApplicationConfig implements WebMvcConfigurer +{ + @Override + public void addInterceptors(InterceptorRegistry registry) + { + registry + .addInterceptor(transformInterceptor()) + .addPathPatterns(ENDPOINT_TRANSFORM, "/live", "/ready"); + } + + @Bean + public TransformInterceptor transformInterceptor() + { + return new TransformInterceptor(); + } + + @Bean + public RestTemplate restTemplate() + { + return new RestTemplate(); + } + + @Bean + public AlfrescoSharedFileStoreClient alfrescoSharedFileStoreClient() + { + return new AlfrescoSharedFileStoreClient(); + } + + @Bean + public TransformRequestValidator transformRequestValidator() + { + return new TransformRequestValidator(); + } + + @Autowired Environment env; + + @Bean + public TransformServiceRegistry transformRegistry() + { + return new TransformRegistryImpl(); + } + + @Bean + public TransformerDebug transformerDebug() + { + return new TransformerDebug().setIsTEngine(true); + } +} diff --git a/t-engine-base/src/main/java/org/alfresco/transform/base/executors/AbstractCommandExecutor.java b/t-engine-base/src/main/java/org/alfresco/transform/base/executors/AbstractCommandExecutor.java new file mode 100644 index 00000000..f48f886a --- /dev/null +++ b/t-engine-base/src/main/java/org/alfresco/transform/base/executors/AbstractCommandExecutor.java @@ -0,0 +1,91 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.base.executors; + +import static org.alfresco.transform.base.executors.RuntimeExec.ExecutionResult; +import static org.springframework.http.HttpStatus.BAD_REQUEST; +import static org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR; + +import java.io.File; +import java.util.Map; + +import org.alfresco.transform.common.TransformException; + +/** + * + */ +public abstract class AbstractCommandExecutor implements CommandExecutor +{ + protected RuntimeExec transformCommand = createTransformCommand(); + protected RuntimeExec checkCommand = createCheckCommand(); + + protected abstract RuntimeExec createTransformCommand(); + + protected abstract RuntimeExec createCheckCommand(); + + @Override + public void run(Map properties, File targetFile, Long timeout) + { + timeout = timeout != null && timeout > 0 ? timeout : 0; + final ExecutionResult result = transformCommand.execute(properties, timeout); + + if (result.getExitValue() != 0 && result.getStdErr() != null && result.getStdErr().length() > 0) + { + throw new TransformException(BAD_REQUEST.value(), + "Transformer exit code was not 0: \n" + result.getStdErr()); + } + + if (!targetFile.exists() || targetFile.length() == 0) + { + throw new TransformException(INTERNAL_SERVER_ERROR.value(), + "Transformer failed to create an output file"); + } + } + + @Override + public String version() + { + if (checkCommand != null) + { + final ExecutionResult result = checkCommand.execute(); + if (result.getExitValue() != 0 && result.getStdErr() != null && result.getStdErr().length() > 0) + { + throw new TransformException(INTERNAL_SERVER_ERROR.value(), + "Transformer version check exit code was not 0: \n" + result); + } + + final String version = result.getStdOut().trim(); + if (version.isEmpty()) + { + throw new TransformException(INTERNAL_SERVER_ERROR.value(), + "Transformer version check failed to create any output"); + } + return version; + } + return "Version not checked"; + } +} diff --git a/t-engine-base/src/main/java/org/alfresco/transform/base/executors/CommandExecutor.java b/t-engine-base/src/main/java/org/alfresco/transform/base/executors/CommandExecutor.java new file mode 100644 index 00000000..45cf3041 --- /dev/null +++ b/t-engine-base/src/main/java/org/alfresco/transform/base/executors/CommandExecutor.java @@ -0,0 +1,71 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.base.executors; + +import org.alfresco.transform.base.logging.LogEntry; + +import java.io.File; +import java.util.HashMap; +import java.util.Map; + +/** + * Basic interface for executing transformations via Shell commands + * + * @author Cezar Leahu + */ +public interface CommandExecutor extends Transformer +{ + void run(Map properties, File targetFile, Long timeout); + + String version(); + + default void run(String options, File sourceFile, File targetFile, + Long timeout) + { + LogEntry.setOptions(options); + + Map properties = new HashMap<>(); + properties.put("options", options); + properties.put("source", sourceFile.getAbsolutePath()); + properties.put("target", targetFile.getAbsolutePath()); + + run(properties, targetFile, timeout); + } + + default void run(String options, File sourceFile, String pageRange, File targetFile, + Long timeout) + { + LogEntry.setOptions(pageRange + (pageRange.isEmpty() ? "" : " ") + options); + + Map properties = new HashMap<>(); + properties.put("options", options); + properties.put("source", sourceFile.getAbsolutePath() + pageRange); + properties.put("target", targetFile.getAbsolutePath()); + + run(properties, targetFile, timeout); + } +} diff --git a/t-engine-base/src/main/java/org/alfresco/transform/base/executors/ExecParameterTokenizer.java b/t-engine-base/src/main/java/org/alfresco/transform/base/executors/ExecParameterTokenizer.java new file mode 100644 index 00000000..313fe597 --- /dev/null +++ b/t-engine-base/src/main/java/org/alfresco/transform/base/executors/ExecParameterTokenizer.java @@ -0,0 +1,360 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.base.executors; + +import static java.util.Collections.singletonList; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.StringTokenizer; + +/** + * DUPLICATED FROM *alfresco-core*. + * + * This class is used to tokenize strings used as parameters for {@link RuntimeExec} objects. + * Examples of such strings are as follows (ImageMagick-like parameters): + *
    + *
  • -font Helvetica -pointsize 50
    • + *
  • -font Helvetica -pointsize 50 -draw "circle 100,100 150,150"
    • + *
  • -font Helvetica -pointsize 50 -draw "gravity south fill black text 0,12 'CopyRight'"
    • + *
+ * The first is the simple case which would be parsed into Strings as follows: + * "-font", "Helvetica", "-pointsize", "50" + *

+ * The second is more complex in that it includes a quoted parameter, which would be parsed as a single String: + * "-font", "Helvetica", "-pointsize", "50", "circle 100,100 150,150" + * Note however that the quotation characters will be stripped from the token. + *

+ * The third shows an example with embedded quotation marks, which would parse to: + * "-font", "Helvetica", "-pointsize", "50", "gravity south fill black text 0,12 'CopyRight'" + * In this case, the embedded quotation marks (which must be different from those surrounding the parameter) + * are preserved in the extracted token. + *

+ * The class does not understand escaped quotes such as p1 p2 "a b c \"hello\" d" p4 + * + * @author Neil Mc Erlean + * @since 3.4.2 + */ +public class ExecParameterTokenizer +{ + /** + * The string to be tokenized. + */ + private final String str; + + /** + * The list of tokens, which will take account of quoted sections. + */ + private List tokens; + + public ExecParameterTokenizer(String str) + { + this.str = str; + } + + /** + * This method returns the tokens in a parameter string. + * Any tokens not contained within single or double quotes will be tokenized in the normal + * way i.e. by using whitespace separators and the standard StringTokenizer algorithm. + * Any tokens which are contained within single or double quotes will be returned as single + * String instances and will have their quote marks removed. + *

+ * See above for examples. + * + * @throws NullPointerException if the string to be tokenized was null. + */ + public List getAllTokens() + { + if (this.str == null) + { + throw new NullPointerException("Illegal null string cannot be tokenized."); + } + + if (tokens == null) + { + tokens = new ArrayList<>(); + + // Preserve original behaviour from RuntimeExec. + if (str.indexOf('\'') == -1 && str.indexOf('"') == -1) + { + // Contains no quotes. + for (StringTokenizer standardTokenizer = new StringTokenizer( + str); standardTokenizer.hasMoreTokens(); ) + { + tokens.add(standardTokenizer.nextToken()); + } + } + else + { + // There are either single or double quotes or both. + // So we need to identify the quoted regions within the string. + List> quotedRegions = new ArrayList<>(); + + for (Pair next = identifyNextQuotedRegion(str, 0); next != null; ) + { + quotedRegions.add(next); + next = identifyNextQuotedRegion(str, next.getSecond() + 1); + } + + // Now we've got a List of index pairs identifying the quoted regions. + // We need to get substrings of quoted and unquoted blocks, whilst maintaining order. + List substrings = getSubstrings(str, quotedRegions); + + for (Substring r : substrings) + { + tokens.addAll(r.getTokens()); + } + } + } + + return this.tokens; + } + + /** + * The substrings will be a list of quoted and unquoted substrings. + * The unquoted ones need to be further tokenized in the normal way. + * The quoted ones must not be tokenized, but need their quotes stripped off. + */ + private List getSubstrings(String str, + List> quotedRegionIndices) + { + List result = new ArrayList<>(); + + int cursorPosition = 0; + for (Pair nextQuotedRegionIndices : quotedRegionIndices) + { + if (cursorPosition < nextQuotedRegionIndices.getFirst()) + { + int startIndexOfNextQuotedRegion = nextQuotedRegionIndices.getFirst() - 1; + result.add(new UnquotedSubstring( + str.substring(cursorPosition, startIndexOfNextQuotedRegion))); + } + result.add(new QuotedSubstring(str.substring(nextQuotedRegionIndices.getFirst(), + nextQuotedRegionIndices.getSecond()))); + cursorPosition = nextQuotedRegionIndices.getSecond(); + } + + // We've processed all the quoted regions, but there may be a final unquoted region + if (cursorPosition < str.length() - 1) + { + result.add(new UnquotedSubstring(str.substring(cursorPosition, str.length() - 1))); + } + + return result; + } + + private Pair identifyNextQuotedRegion(String str, int startingIndex) + { + int indexOfNextSingleQuote = str.indexOf('\'', startingIndex); + int indexOfNextDoubleQuote = str.indexOf('"', startingIndex); + + if (indexOfNextSingleQuote == -1 && indexOfNextDoubleQuote == -1) + { + // If there are no more quoted regions + return null; + } + else if (indexOfNextSingleQuote > -1 && indexOfNextDoubleQuote > -1) + { + // If there are both single and double quotes in the remainder of the string + // Then select the closest quote. + int indexOfNextQuote = Math.min(indexOfNextSingleQuote, indexOfNextDoubleQuote); + char quoteChar = str.charAt(indexOfNextQuote); + + return findIndexOfClosingQuote(str, indexOfNextQuote, quoteChar); + } + else + { + // Only one of the quote characters is present. + + int indexOfNextQuote = Math.max(indexOfNextSingleQuote, indexOfNextDoubleQuote); + char quoteChar = str.charAt(indexOfNextQuote); + + return findIndexOfClosingQuote(str, indexOfNextQuote, quoteChar); + } + } + + private Pair findIndexOfClosingQuote(String str, int indexOfStartingQuote, + char quoteChar) + { + // So we know which type of quote char we're dealing with. Either ' or ". + // Now we need to find the closing quote. + int indexAfterClosingQuote = str.indexOf(quoteChar, + indexOfStartingQuote + 1) + 1; // + 1 to search after opening quote. + 1 to give result including closing quote. + + if (indexAfterClosingQuote == 0) // -1 + 1 + { + // If no closing quote. + throw new IllegalArgumentException("No closing " + quoteChar + "quote in" + str); + } + + return new Pair<>(indexOfStartingQuote, indexAfterClosingQuote); + } + + /** + * Utility interface for a substring in a parameter string. + */ + public interface Substring + { + /** + * Gets all the tokens in a parameter string. + */ + List getTokens(); + } + + /** + * A substring that is not surrounded by (single or double) quotes. + */ + public class UnquotedSubstring implements Substring + { + private final String regionString; + + public UnquotedSubstring(String str) + { + this.regionString = str; + } + + public List getTokens() + { + StringTokenizer t = new StringTokenizer(regionString); + List result = new ArrayList<>(); + while (t.hasMoreTokens()) + { + result.add(t.nextToken()); + } + return result; + } + + public String toString() + { + return UnquotedSubstring.class.getSimpleName() + ": '" + regionString + '\''; + } + } + + /** + * A substring that is surrounded by (single or double) quotes. + */ + public class QuotedSubstring implements Substring + { + private final String regionString; + + public QuotedSubstring(String str) + { + this.regionString = str; + } + + public List getTokens() + { + String stringWithoutQuotes = regionString.substring(1, regionString.length() - 1); + return singletonList(stringWithoutQuotes); + } + + public String toString() + { + return QuotedSubstring.class.getSimpleName() + ": '" + regionString + '\''; + } + } + + public static final class Pair implements Serializable + { + private static final long serialVersionUID = -7406248421185630612L; + + /** + * The first member of the pair. + */ + private F first; + + /** + * The second member of the pair. + */ + private S second; + + /** + * Make a new one. + * + * @param first The first member. + * @param second The second member. + */ + public Pair(F first, S second) + { + this.first = first; + this.second = second; + } + + /** + * Get the first member of the tuple. + * + * @return The first member. + */ + public final F getFirst() + { + return first; + } + + /** + * Get the second member of the tuple. + * + * @return The second member. + */ + public final S getSecond() + { + return second; + } + + public final void setFirst(F first) + { + this.first = first; + } + + public final void setSecond(S second) + { + this.second = second; + } + + @Override public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Pair pair = (Pair) o; + return Objects.equals(first, pair.first) && + Objects.equals(second, pair.second); + } + + @Override public int hashCode() + { + return Objects.hash(first, second); + } + + @Override + public String toString() + { + return "(" + first + ", " + second + ")"; + } + } +} + diff --git a/t-engine-base/src/main/java/org/alfresco/transform/base/executors/JavaExecutor.java b/t-engine-base/src/main/java/org/alfresco/transform/base/executors/JavaExecutor.java new file mode 100644 index 00000000..545a2f27 --- /dev/null +++ b/t-engine-base/src/main/java/org/alfresco/transform/base/executors/JavaExecutor.java @@ -0,0 +1,40 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.base.executors; + +import java.io.File; + +/** + * Basic interface for executing transformations inside Java/JVM. + * + * @author Cezar Leahu + * @author adavis + */ +public interface JavaExecutor extends Transformer +{ + void call(File sourceFile, File targetFile, String... args) throws Exception; +} diff --git a/t-engine-base/src/main/java/org/alfresco/transform/base/executors/RuntimeExec.java b/t-engine-base/src/main/java/org/alfresco/transform/base/executors/RuntimeExec.java new file mode 100644 index 00000000..3363f646 --- /dev/null +++ b/t-engine-base/src/main/java/org/alfresco/transform/base/executors/RuntimeExec.java @@ -0,0 +1,986 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.base.executors; + +import static java.util.Collections.emptyMap; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.nio.charset.UnsupportedCharsetException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.StringTokenizer; +import java.util.Timer; +import java.util.TimerTask; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * DUPLICATED FROM *alfresco-core*. + * + * This acts as a session similar to the java.lang.Process, but + * logs the system standard and error streams. + *

+ * The bean can be configured to execute a command directly, or be given a map + * of commands keyed by the os.name Java system property. In this map, + * the default key that is used when no match is found is the + * {@link #KEY_OS_DEFAULT *} key. + *

+ * Use the {@link #setProcessDirectory(String) processDirectory} property to change the default location + * from which the command executes. The process's environment can be configured using the + * {@link #setProcessProperties(Map) processProperties} property. + *

+ * Commands may use placeholders, e.g. + * 


+ *    find
+ *    -name
+ *    ${filename}
+ *
+ * The filename property will be substituted for any supplied value prior to + * each execution of the command. Currently, no checks are made to get or check the + * properties contained within the command string. It is up to the client code to + * dynamically extract the properties required if the required properties are not + * known up front. + *

+ * Sometimes, a variable may contain several arguments. . In this case, the arguments + * need to be tokenized using a standard StringTokenizer. To force tokenization + * of a value, use: + * 


+ *    SPLIT:${userArgs}
+ *
+ * You should not use this just to split up arguments that are known to require tokenization + * up front. The SPLIT: directive works for the entire argument and will not do anything + * if it is not at the beginning of the argument. Do not use SPLIT: to break up arguments + * that are fixed, so avoid doing this: + * 

+ *    SPLIT:ls -lih
+ *
+ * Instead, break the command up explicitly: + * 

+ *    ls
+ *    -lih
+ *
+ * + * Tokenization of quoted parameter values is handled by ExecParameterTokenizer, which + * describes the support in more detail. + * + * @author Derek Hulley + */ +public class RuntimeExec +{ + private static final Logger logger = LoggerFactory.getLogger(RuntimeExec.class); + + /** + * the key to use when specifying a command for any other OS: * + */ + private static final String KEY_OS_DEFAULT = "*"; + + private static final String KEY_OS_NAME = "os.name"; + private static final int BUFFER_SIZE = 1024; + private static final String VAR_OPEN = "${"; + private static final String VAR_CLOSE = "}"; + private static final String DIRECTIVE_SPLIT = "SPLIT:"; + + private String[] command; + private Charset charset = Charset.defaultCharset(); + private boolean waitForCompletion = true; + private Map defaultProperties = emptyMap(); + private String[] processProperties; + private File processDirectory; + private final Set errCodes; + private final Timer timer = new Timer(true); + + /** + * Default constructor. Initialize this instance by setting individual properties. + */ + public RuntimeExec() + { + // set default error codes + errCodes = new HashSet<>(2); + errCodes.add(1); + errCodes.add(2); + } + + public String toString() + { + final StringBuilder sb = new StringBuilder(256); + sb.append("RuntimeExec:\n").append(" command: "); + if (command == null) + { + // command is 'null', so there's nothing to toString + sb.append("'null'\n"); + } + else + { + for (String cmdStr : command) + { + sb.append(cmdStr).append(" "); + } + sb.append("\n"); + } + sb.append(" env props: ").append(Arrays.toString(processProperties)).append("\n") + .append(" dir: ").append(processDirectory).append("\n") + .append(" os: ").append(System.getProperty(KEY_OS_NAME)).append("\n"); + return sb.toString(); + } + + /** + * Set the command to execute regardless of operating system + * + * @param command an array of strings representing the command (first entry) and arguments + * @since 3.0 + */ + public void setCommand(String[] command) + { + this.command = command; + } + + /** + * Sets the assumed charset of OUT and ERR streams generated by the executed command. + * This defaults to the system default charset: {@link Charset#defaultCharset()}. + * + * @param charsetCode a supported character set code + * @throws UnsupportedCharsetException if the characterset code is not recognised by Java + */ + public void setCharset(String charsetCode) + { + this.charset = Charset.forName(charsetCode); + } + + /** + * Set whether to wait for completion of the command or not. If there is no wait for completion, + * then the return value of out and err buffers cannot be relied upon as the + * command may still be in progress. Failure is therefore not possible unless the calling thread + * waits for execution. + * + * @param waitForCompletion true (default) is to wait for the command to exit, + * or false to just return an exit code of 0 and whatever + * output is available at that point. + * @since 2.1 + */ + public void setWaitForCompletion(boolean waitForCompletion) + { + this.waitForCompletion = waitForCompletion; + } + + /** + * Supply a choice of commands to execute based on a mapping from the os.name system + * property to the command to execute. The {@link #KEY_OS_DEFAULT *} key can be used + * to get a command where there is not direct match to the operating system key. + *

+ * Each command is an array of strings, the first of which represents the command and all subsequent + * entries in the array represent the arguments. All elements of the array will be checked for + * the presence of any substitution parameters (e.g. '{dir}'). The parameters can be set using the + * {@link #setDefaultProperties(Map) defaults} or by passing the substitution values into the + * {@link #execute(Map)} command. + *

+ * If parameters passed may be multiple arguments, or if the values provided in the map are themselves + * collections of arguments (not recommended), then prefix the value with SPLIT: to ensure that + * the value is tokenized before being passed to the command. Any values that are not split, will be + * passed to the command as single arguments. For example:
+ * 'SPLIT: dir . ..' becomes 'dir', '.' and '..'.
+ * 'SPLIT: dir ${path}' (if path is '. ..') becomes 'dir', '.' and '..'.
+ * The splitting occurs post-subtitution. Where the arguments are known, it is advisable to avoid + * SPLIT:. + * + * @param commandsByOS a map of command string arrays, keyed by operating system names + * @see #setDefaultProperties(Map) + * @since 3.0 + */ + public void setCommandsAndArguments(Map commandsByOS) + { + // get the current OS + String serverOs = System.getProperty(KEY_OS_NAME); + // attempt to find a match + String[] command = commandsByOS.get(serverOs); + if (command == null) + { + // go through the commands keys, looking for one that matches by regular expression matching + for (String osName : commandsByOS.keySet()) + { + // Ignore * options. It is dealt with later. + if (osName.equals(KEY_OS_DEFAULT)) + { + continue; + } + // Do regex match + if (serverOs.matches(osName)) + { + command = commandsByOS.get(osName); + break; + } + } + // if there is still no command, then check for the wildcard + if (command == null) + { + command = commandsByOS.get(KEY_OS_DEFAULT); + } + } + // check + if (command == null) + { + throw new RuntimeException( + "No command found for OS " + serverOs + " or '" + KEY_OS_DEFAULT + "': \n" + + " commands: " + commandsByOS); + } + this.command = command; + } + + /** + * Supply a choice of commands to execute based on a mapping from the os.name system + * property to the command to execute. The {@link #KEY_OS_DEFAULT *} key can be used + * to get a command where there is not direct match to the operating system key. + * + * @param commandsByOS a map of command string keyed by operating system names + * @deprecated Use {@link #setCommandsAndArguments(Map)} + */ + public void setCommandMap(Map commandsByOS) + { + // This is deprecated, so issue a warning + logger.warn( + "The bean RuntimeExec property 'commandMap' has been deprecated;" + + " use 'commandsAndArguments' instead. See https://issues.alfresco.com/jira/browse/ETHREEOH-579."); + Map fixed = new LinkedHashMap<>(); + for (Map.Entry entry : commandsByOS.entrySet()) + { + String os = entry.getKey(); + String unparsedCmd = entry.getValue(); + StringTokenizer tokenizer = new StringTokenizer(unparsedCmd); + String[] cmd = new String[tokenizer.countTokens()]; + for (int i = 0; i < cmd.length; i++) + { + cmd[i] = tokenizer.nextToken(); + } + fixed.put(os, cmd); + } + setCommandsAndArguments(fixed); + } + + /** + * Set the default command-line properties to use when executing the command. + * These are properties that substitute variables defined in the command string itself. + * Properties supplied during execution will overwrite the default properties. + *

+ * null properties will be treated as an empty string for substitution + * purposes. + * + * @param defaultProperties property values + */ + public void setDefaultProperties(Map defaultProperties) + { + this.defaultProperties = defaultProperties; + } + + /** + * Set additional runtime properties (environment properties) that will used + * by the executing process. + *

+ * Any keys or properties that start and end with ${...} will be removed on the assumption + * that these are unset properties. null values are translated to empty strings. + * All keys and values are trimmed of leading and trailing whitespace. + * + * @param processProperties Runtime process properties + * @see Runtime#exec(String, String[], java.io.File) + */ + public void setProcessProperties(Map processProperties) + { + ArrayList processPropList = new ArrayList<>(processProperties.size()); + boolean hasPath = false; + String systemPath = System.getenv("PATH"); + for (Map.Entry entry : processProperties.entrySet()) + { + String key = entry.getKey(); + String value = entry.getValue(); + if (key == null) + { + continue; + } + if (value == null) + { + value = ""; + } + key = key.trim(); + value = value.trim(); + if (key.startsWith(VAR_OPEN) && key.endsWith(VAR_CLOSE)) + { + continue; + } + if (value.startsWith(VAR_OPEN) && value.endsWith(VAR_CLOSE)) + { + continue; + } + // If a path is specified, prepend it to the existing path + if (key.equals("PATH")) + { + if (systemPath != null && systemPath.length() > 0) + { + processPropList.add(key + "=" + value + File.pathSeparator + systemPath); + } + else + { + processPropList.add(key + "=" + value); + } + hasPath = true; + } + else + { + processPropList.add(key + "=" + value); + } + } + // If a path was not specified, inherit the current one + if (!hasPath && systemPath != null && systemPath.length() > 0) + { + processPropList.add("PATH=" + systemPath); + } + this.processProperties = processPropList.toArray(new String[0]); + } + + /** + * Adds a property to existed processProperties. + * Property should not be null or empty. + * If property with the same value already exists then no change is made. + * If property exists with a different value then old value is replaced with the new one. + * + * @param name - property name + * @param value - property value + */ + public void setProcessProperty(String name, String value) + { + boolean set = false; + + if (name == null || value == null) + { + return; + } + + name = name.trim(); + value = value.trim(); + + if (name.isEmpty() || value.isEmpty()) + { + return; + } + + String property = name + "=" + value; + + for (String prop : this.processProperties) + { + if (prop.equals(property)) + { + set = true; + break; + } + + if (prop.startsWith(name)) + { + String oldValue = prop.split("=")[1]; + prop.replace(oldValue, value); + set = true; + } + } + + if (!set) + { + String[] existedProperties = this.processProperties; + int epl = existedProperties.length; + String[] newProperties = Arrays.copyOf(existedProperties, epl + 1); + newProperties[epl] = property; + this.processProperties = newProperties; + } + } + + /** + * Set the runtime location from which the command is executed. + *

+ * If the value is an unsubsititued variable (${...}) then it is ignored. + * If the location is not visible at the time of setting, a warning is issued only. + * + * @param processDirectory the runtime location from which to execute the command + */ + public void setProcessDirectory(String processDirectory) + { + if (processDirectory.startsWith(VAR_OPEN) && processDirectory.endsWith(VAR_CLOSE)) + { + this.processDirectory = null; + } + else + { + this.processDirectory = new File(processDirectory); + if (!this.processDirectory.exists()) + { + logger.warn( + "The runtime process directory is not visible when setting property " + + "'processDirectory': \n{}", this); + } + } + } + + /** + * A comma or space separated list of values that, if returned by the executed command, + * indicate an error value. This defaults to "1, 2". + * + * @param errCodesStr the error codes for the execution + */ + public void setErrorCodes(String errCodesStr) + { + errCodes.clear(); + StringTokenizer tokenizer = new StringTokenizer(errCodesStr, " ,"); + while (tokenizer.hasMoreElements()) + { + String errCodeStr = tokenizer.nextToken(); + // attempt to convert it to an integer + try + { + int errCode = Integer.parseInt(errCodeStr); + this.errCodes.add(errCode); + } + catch (NumberFormatException e) + { + throw new RuntimeException( + "Property 'errorCodes' must be comma-separated list of integers: " + errCodesStr); + } + } + } + + /** + * Executes the command using the default properties + * + * @see #execute(Map) + */ + public ExecutionResult execute() + { + return execute(defaultProperties); + } + + /** + * Executes the statement that this instance was constructed with. + * + * @param properties the properties that the command might be executed with. + * null properties will be treated as an empty string for substitution + * purposes. + * @return Returns the full execution results + */ + public ExecutionResult execute(Map properties) + { + return execute(properties, -1); + } + + /** + * Executes the statement that this instance was constructed with an optional + * timeout after which the command is asked to + * + * @param properties the properties that the command might be executed with. + * null properties will be treated as an empty string for substitution + * purposes. + * @param timeoutMs a timeout after which {@link Process#destroy()} is called. + * ignored if less than or equal to zero. Note this method does not guarantee + * to terminate the process (it is not a kill -9). + * @return Returns the full execution results + */ + public ExecutionResult execute(Map properties, final long timeoutMs) + { + int defaultFailureExitValue = errCodes.size() > 0 ? ((Integer) errCodes.toArray()[0]) : 1; + + // check that the command has been set + if (command == null) + { + throw new RuntimeException("Runtime command has not been set: \n" + this); + } + + // create the properties + Runtime runtime = Runtime.getRuntime(); + Process process; + String[] commandToExecute = null; + try + { + // execute the command with full property replacement + commandToExecute = getCommand(properties); + final Process thisProcess = runtime.exec(commandToExecute, processProperties, + processDirectory); + process = thisProcess; + if (timeoutMs > 0) + { + final String[] command = commandToExecute; + timer.schedule(new TimerTask() + { + @Override + public void run() + { + // Only try to kill the process if it is still running + try + { + thisProcess.exitValue(); + } + catch (IllegalThreadStateException stillRunning) + { + logger.debug( + "Process has taken too long ({} seconds). Killing process {}", + timeoutMs / 1000, Arrays.deepToString(command)); + } + } + }, timeoutMs); + } + } + catch (IOException e) + { + // The process could not be executed here, so just drop out with an appropriate error state + String execOut = ""; + String execErr = e.getMessage(); + ExecutionResult result = new ExecutionResult(null, commandToExecute, errCodes, + defaultFailureExitValue, execOut, execErr); + logFullEnvironmentDump(result); + return result; + } + + // create the stream gobblers + InputStreamReaderThread stdOutGobbler = new InputStreamReaderThread( + process.getInputStream(), charset); + InputStreamReaderThread stdErrGobbler = new InputStreamReaderThread( + process.getErrorStream(), charset); + + // start gobbling + stdOutGobbler.start(); + stdErrGobbler.start(); + + // wait for the process to finish + int exitValue = 0; + try + { + if (waitForCompletion) + { + exitValue = process.waitFor(); + } + } + catch (InterruptedException e) + { + // process was interrupted - generate an error message + stdErrGobbler.addToBuffer(e.toString()); + exitValue = defaultFailureExitValue; + } + + if (waitForCompletion) + { + // ensure that the stream gobblers get to finish + stdOutGobbler.waitForCompletion(); + stdErrGobbler.waitForCompletion(); + } + + // get the stream values + String execOut = stdOutGobbler.getBuffer(); + String execErr = stdErrGobbler.getBuffer(); + + // construct the return value + ExecutionResult result = new ExecutionResult(process, commandToExecute, errCodes, exitValue, + execOut, execErr); + + // done + logFullEnvironmentDump(result); + return result; + } + + /** + * Dump the full environment in debug mode + */ + private void logFullEnvironmentDump(ExecutionResult result) + { + if (logger.isTraceEnabled()) + { + StringBuilder sb = new StringBuilder(); + sb.append(result); + + // Environment variables modified by Alfresco + if (processProperties != null && processProperties.length > 0) + { + sb.append("\n modified environment: "); + for (String property : processProperties) + { + sb.append("\n "); + sb.append(property); + } + } + + // Dump the full environment + sb.append("\n existing environment: "); + Map envVariables = System.getenv(); + for (Map.Entry entry : envVariables.entrySet()) + { + String name = entry.getKey(); + String value = entry.getValue(); + sb.append("\n "); + sb.append(name).append("=").append(value); + } + + logger.trace(sb.toString()); + } + logger.debug("Result: " + result.toString()); + + // close output stream (connected to input stream of native subprocess) + } + + /** + * @return Returns the command that will be executed if no additional properties + * were to be supplied + */ + public String[] getCommand() + { + return getCommand(defaultProperties); + } + + /** + * Get the command that will be executed post substitution. + *

+ * null properties will be treated as an empty string for substitution + * purposes. + * + * @param properties the properties that the command might be executed with + * @return Returns the command that will be executed should the additional properties + * be supplied + */ + public String[] getCommand(Map properties) + { + Map execProperties; + if (properties == defaultProperties) + { + // we are just using the default properties + execProperties = defaultProperties; + } + else + { + execProperties = new HashMap<>(defaultProperties); + // overlay the supplied properties + execProperties.putAll(properties); + } + // Perform the substitution for each element of the command + ArrayList adjustedCommandElements = new ArrayList<>(20); + for (String s : command) + { + StringBuilder sb = new StringBuilder(s); + for (Map.Entry entry : execProperties.entrySet()) + { + String key = entry.getKey(); + String value = entry.getValue(); + // ignore null + if (value == null) + { + value = ""; + } + // progressively replace the property in the command + key = (VAR_OPEN + key + VAR_CLOSE); + int index = sb.indexOf(key); + while (index > -1) + { + // replace + sb.replace(index, index + key.length(), value); + // get the next one + index = sb.indexOf(key, index + 1); + } + } + String adjustedValue = sb.toString(); + // Now SPLIT: it + if (adjustedValue.startsWith(DIRECTIVE_SPLIT)) + { + String unsplitAdjustedValue = sb.substring(DIRECTIVE_SPLIT.length()); + + // There may be quoted arguments here (see ALF-7482) + ExecParameterTokenizer quoteAwareTokenizer = new ExecParameterTokenizer( + unsplitAdjustedValue); + List tokens = quoteAwareTokenizer.getAllTokens(); + adjustedCommandElements.addAll(tokens); + } + else + { + adjustedCommandElements.add(adjustedValue); + } + } + // done + return adjustedCommandElements.toArray(new String[0]); + } + + /** + * Object to carry the results of an execution to the caller. + * + * @author Derek Hulley + */ + public static class ExecutionResult + { + private final Process process; + private final String[] command; + private final Set errCodes; + private final int exitValue; + private final String stdOut; + private final String stdErr; + + /** + * @param process the process attached to Java - null is allowed + */ + private ExecutionResult( + final Process process, + final String[] command, + final Set errCodes, + final int exitValue, + final String stdOut, + final String stdErr) + { + this.process = process; + this.command = command; + this.errCodes = errCodes; + this.exitValue = exitValue; + this.stdOut = stdOut; + this.stdErr = stdErr; + } + + @Override + public String toString() + { + String out = stdOut.length() > 250 ? stdOut.substring(0, 250) : stdOut; + String err = stdErr.length() > 250 ? stdErr.substring(0, 250) : stdErr; + + StringBuilder sb = new StringBuilder(128); + sb.append("Execution result: \n") + .append(" os: ").append(System.getProperty(KEY_OS_NAME)).append("\n") + .append(" command: "); + appendCommand(sb, command).append("\n") + .append(" succeeded: ").append(getSuccess()).append("\n") + .append(" exit code: ").append(exitValue).append("\n") + .append(" out: ").append(out).append("\n") + .append(" err: ").append(err); + return sb.toString(); + } + + /** + * Appends the command in a form that make running from the command line simpler. + * It is not a real attempt at making a command given all the operating system + * and shell options, but makes copy, paste and edit a bit simpler. + */ + private StringBuilder appendCommand(StringBuilder sb, String[] command) + { + boolean arg = false; + for (String element : command) + { + if (element == null) + { + continue; + } + + if (arg) + { + sb.append(' '); + } + else + { + arg = true; + } + + boolean escape = element.indexOf(' ') != -1 || element.indexOf('>') != -1; + if (escape) + { + sb.append("\""); + } + sb.append(element); + if (escape) + { + sb.append("\""); + } + } + return sb; + } + + /** + * A helper method to force a kill of the process that generated this result. This is + * useful in cases where the process started is not expected to exit, or doesn't exit + * quickly. If the {@linkplain RuntimeExec#setWaitForCompletion(boolean) "wait for completion"} + * flag is false then the process may still be running when this result is returned. + * + * @return true if the process was killed, otherwise false + */ + public boolean killProcess() + { + if (process == null) + { + return true; + } + try + { + process.destroy(); + return true; + } + catch (Throwable e) + { + logger.warn(e.getMessage()); + return false; + } + } + + /** + * @param exitValue the command exit value + * @return Returns true if the code is a listed failure code + * @see #setErrorCodes(String) + */ + private boolean isFailureCode(int exitValue) + { + return errCodes.contains(exitValue); + } + + /** + * @return Returns true if the command was deemed to be successful according to the + * failure codes returned by the execution. + */ + public boolean getSuccess() + { + return !isFailureCode(exitValue); + } + + public int getExitValue() + { + return exitValue; + } + + public String getStdOut() + { + return stdOut; + } + + public String getStdErr() + { + return stdErr; + } + } + + /** + * Gobbles an InputStream and writes it into a + * StringBuffer + *

+ * The reading of the input stream is buffered. + */ + public static class InputStreamReaderThread extends Thread + { + private final InputStream is; + private final Charset charset; + private final StringBuffer buffer; // we require the synchronization + private boolean completed; + + /** + * @param is an input stream to read - it will be wrapped in a buffer + * for reading + */ + public InputStreamReaderThread(InputStream is, Charset charset) + { + super(); + setDaemon(true); // must not hold up the VM if it is terminating + this.is = is; + this.charset = charset; + this.buffer = new StringBuffer(BUFFER_SIZE); + this.completed = false; + } + + public synchronized void run() + { + completed = false; + + byte[] bytes = new byte[BUFFER_SIZE]; + try (InputStream tempIs = new BufferedInputStream(is, BUFFER_SIZE)) + { + int count = -2; + while (count != -1) + { + // do we have something previously read? + if (count > 0) + { + String toWrite = new String(bytes, 0, count, charset.name()); + buffer.append(toWrite); + } + // read the next set of bytes + count = tempIs.read(bytes); + } + // done + } + catch (IOException e) + { + throw new RuntimeException("Unable to read stream", e); + } + finally + { + // The thread has finished consuming the stream + completed = true; + // Notify waiters + this.notifyAll(); // Note: Method is synchronized + } + } + + /** + * Waits for the run to complete. + *

+ * Remember to start the thread first + */ + public synchronized void waitForCompletion() + { + while (!completed) + { + try + { + // release our lock and wait a bit + this.wait(1000L); // 200 ms + } + catch (InterruptedException ignore) + { + } + } + } + + /** + * @param msg the message to add to the buffer + */ + public void addToBuffer(String msg) + { + buffer.append(msg); + } + + public boolean isComplete() + { + return completed; + } + + /** + * @return Returns the current state of the buffer + */ + public String getBuffer() + { + return buffer.toString(); + } + } +} + diff --git a/t-engine-base/src/main/java/org/alfresco/transform/base/executors/Transformer.java b/t-engine-base/src/main/java/org/alfresco/transform/base/executors/Transformer.java new file mode 100644 index 00000000..73751b94 --- /dev/null +++ b/t-engine-base/src/main/java/org/alfresco/transform/base/executors/Transformer.java @@ -0,0 +1,130 @@ +package org.alfresco.transform.base.executors; + +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ + +import org.alfresco.transform.common.TransformException; + +import java.io.File; +import java.util.Map; + +import static org.alfresco.transform.common.Mimetype.MIMETYPE_METADATA_EMBED; +import static org.alfresco.transform.common.Mimetype.MIMETYPE_METADATA_EXTRACT; +import static org.alfresco.transform.base.util.RequestParamMap.TRANSFORM_NAME_PARAMETER; +import static org.springframework.http.HttpStatus.BAD_REQUEST; +import static org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR; + +/** + * Basic interface for executing transforms and metadata extract or embed actions. + * + * @author adavis + */ +public interface Transformer +{ + /** + * @return A unique transformer id, + * + */ + String getTransformerId(); + + default void transform(String sourceMimetype, String targetMimetype, Map transformOptions, + File sourceFile, File targetFile) throws TransformException { + final String transformName = transformOptions.remove(TRANSFORM_NAME_PARAMETER); + transformExtractOrEmbed(transformName, sourceMimetype, targetMimetype, transformOptions, sourceFile, targetFile); + } + + default void transformExtractOrEmbed(String transformName, String sourceMimetype, String targetMimetype, + Map transformOptions, + File sourceFile, File targetFile) throws TransformException + { + try + { + if (MIMETYPE_METADATA_EXTRACT.equals(targetMimetype)) + { + extractMetadata(transformName, sourceMimetype, targetMimetype, transformOptions, sourceFile, targetFile); + } + else if (MIMETYPE_METADATA_EMBED.equals(targetMimetype)) + { + embedMetadata(transformName, sourceMimetype, targetMimetype, transformOptions, sourceFile, targetFile); + } + else + { + transform(transformName, sourceMimetype, targetMimetype, transformOptions, sourceFile, targetFile); + } + } + catch (TransformException e) + { + throw e; + } + catch (IllegalArgumentException e) + { + throw new TransformException(BAD_REQUEST.value(), getMessage(e), e); + } + catch (Exception e) + { + throw new TransformException(INTERNAL_SERVER_ERROR.value(), getMessage(e), e); + } + if (!targetFile.exists()) + { + throw new TransformException(INTERNAL_SERVER_ERROR.value(), + "Transformer failed to create an output file. Target file does not exist."); + } + if (sourceFile.length() > 0 && targetFile.length() == 0) + { + throw new TransformException(INTERNAL_SERVER_ERROR.value(), + "Transformer failed to create an output file. Target file is empty but source file was not empty."); + } + } + + private static String getMessage(Exception e) + { + return e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage(); + } + + default void transform(String transformName, String sourceMimetype, String targetMimetype, + Map transformOptions, + File sourceFile, File targetFile) throws Exception + { + } + + default void extractMetadata(String transformName, String sourceMimetype, String targetMimetype, + Map transformOptions, + File sourceFile, File targetFile) throws Exception + { + } + + /** + * @deprecated The content repository has no non test embed metadata implementations. + * This code exists in case there are custom implementations, that need to be converted to T-Engines. + * It is simply a copy and paste from the content repository and has received limited testing. + */ + default void embedMetadata(String transformName, String sourceMimetype, String targetMimetype, + Map transformOptions, + File sourceFile, File targetFile) throws Exception + { + } +} diff --git a/t-engine-base/src/main/java/org/alfresco/transform/base/fs/FileManager.java b/t-engine-base/src/main/java/org/alfresco/transform/base/fs/FileManager.java new file mode 100644 index 00000000..270d423b --- /dev/null +++ b/t-engine-base/src/main/java/org/alfresco/transform/base/fs/FileManager.java @@ -0,0 +1,249 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.base.fs; + +import static org.springframework.http.HttpHeaders.CONTENT_DISPOSITION; +import static org.springframework.http.HttpStatus.BAD_REQUEST; +import static org.springframework.http.HttpStatus.INSUFFICIENT_STORAGE; +import static org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR; +import static org.springframework.util.StringUtils.getFilename; +import static org.springframework.util.StringUtils.getFilenameExtension; + +import java.io.File; +import java.io.IOException; +import java.net.MalformedURLException; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; +import java.util.Arrays; + +import javax.servlet.http.HttpServletRequest; + +import org.alfresco.transform.common.ExtensionService; +import org.alfresco.transform.common.TransformException; +import org.alfresco.transform.base.logging.LogEntry; +import org.springframework.core.io.Resource; +import org.springframework.core.io.UrlResource; +import org.springframework.http.HttpHeaders; +import org.springframework.http.ResponseEntity; +import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.util.UriUtils; + +public class FileManager +{ + public static final String SOURCE_FILE = "sourceFile"; + public static final String TARGET_FILE = "targetFile"; + private static final String FILENAME = "filename="; + + public static File createTargetFile(HttpServletRequest request, String filename) + { + File file = buildFile(filename); + request.setAttribute(TARGET_FILE, file); + return file; + } + + public static File buildFile(String filename) + { + filename = checkFilename(false, filename); + LogEntry.setTarget(filename); + return TempFileProvider.createTempFile("target_", "_" + filename); + } + + public static void deleteFile(final File file) throws Exception + { + if (!file.delete()) + { + throw new Exception("Failed to delete file"); + } + } + + private static String checkFilename(boolean source, String filename) + { + filename = getFilename(filename); + if (filename == null || filename.isEmpty()) + { + String sourceOrTarget = source ? "source" : "target"; + int statusCode = source ? BAD_REQUEST.value() : INTERNAL_SERVER_ERROR.value(); + throw new TransformException(statusCode, + "The " + sourceOrTarget + " filename was not supplied"); + } + return filename; + } + + private static void save(MultipartFile multipartFile, File file) + { + try + { + Files.copy(multipartFile.getInputStream(), file.toPath(), + StandardCopyOption.REPLACE_EXISTING); + } + catch (IOException e) + { + throw new TransformException(INSUFFICIENT_STORAGE.value(), + "Failed to store the source file", e); + } + } + + public static void save(Resource body, File file) + { + try + { + Files.copy(body.getInputStream(), file.toPath(), StandardCopyOption.REPLACE_EXISTING); + } + catch (IOException e) + { + throw new TransformException(INSUFFICIENT_STORAGE.value(), + "Failed to store the source file", e); + } + } + + private static Resource load(File file) + { + try + { + Resource resource = new UrlResource(file.toURI()); + if (resource.exists() || resource.isReadable()) + { + return resource; + } + else + { + throw new TransformException(INTERNAL_SERVER_ERROR.value(), + "Could not read the target file: " + file.getPath()); + } + } + catch (MalformedURLException e) + { + throw new TransformException(INTERNAL_SERVER_ERROR.value(), + "The target filename was malformed: " + file.getPath(), e); + } + } + + public static String getFilenameFromContentDisposition(HttpHeaders headers) + { + String filename = ""; + String contentDisposition = headers.getFirst(CONTENT_DISPOSITION); + if (contentDisposition != null) + { + String[] strings = contentDisposition.split("; *"); + filename = Arrays.stream(strings) + .filter(s -> s.startsWith(FILENAME)) + .findFirst() + .map(s -> s.substring(FILENAME.length())) + .orElse(""); + } + return filename; + } + + public static String createTargetFileName(final String fileName, String sourceMimetype, String targetMimetype) + { + String targetExtension = ExtensionService.getExtensionForTargetMimetype(targetMimetype, sourceMimetype); + final String sourceFilename = getFilename(fileName); + + if (sourceFilename == null || sourceFilename.isEmpty()) + { + return null; + } + + final String ext = getFilenameExtension(sourceFilename); + if (ext == null || ext.isEmpty()) + { + return sourceFilename + '.' + targetExtension; + } + return sourceFilename.substring(0, sourceFilename.length() - ext.length() - 1) + '.' + targetExtension; + } + + public static File createSourceFile(HttpServletRequest request, MultipartFile multipartFile) + { + String filename = multipartFile.getOriginalFilename(); + long size = multipartFile.getSize(); + filename = checkFilename(true, filename); + File file = TempFileProvider.createTempFile("source_", "_" + filename); + request.setAttribute(SOURCE_FILE, file); + save(multipartFile, file); + LogEntry.setSource(filename, size); + return file; + } + + @SuppressWarnings("ResultOfMethodCallIgnored") + public static void deleteFile(HttpServletRequest request, String attributeName) + { + File file = (File) request.getAttribute(attributeName); + if (file != null) + { + file.delete(); + } + } + + public static ResponseEntity createAttachment(String targetFilename, File + targetFile) + { + Resource targetResource = load(targetFile); + targetFilename = UriUtils.encodePath(getFilename(targetFilename), "UTF-8"); + return ResponseEntity.ok().header(CONTENT_DISPOSITION, + "attachment; filename*= UTF-8''" + targetFilename).body(targetResource); + } + + /** + * TempFileProvider - Duplicated and adapted from alfresco-core. + */ + public static class TempFileProvider + { + public static File createTempFile(final String prefix, final String suffix) + { + final File directory = getTempDir(); + try + { + return File.createTempFile(prefix, suffix, directory); + } + catch (IOException e) + { + throw new RuntimeException( + "Failed to created temp file: \n prefix: " + prefix + + "\n suffix: " + suffix + "\n directory: " + directory, e); + } + } + + private static File getTempDir() + { + final String dirName = "Alfresco"; + final String systemTempDirPath = System.getProperty("java.io.tmpdir"); + if (systemTempDirPath == null) + { + throw new RuntimeException("System property not available: java.io.tmpdir"); + } + + final File systemTempDir = new File(systemTempDirPath); + final File tempDir = new File(systemTempDir, dirName); + if (!tempDir.exists() && !tempDir.mkdirs() && !tempDir.exists()) + { + throw new RuntimeException("Failed to create temp directory: " + tempDir); + } + + return tempDir; + } + } +} diff --git a/t-engine-base/src/main/java/org/alfresco/transform/base/logging/LogEntry.java b/t-engine-base/src/main/java/org/alfresco/transform/base/logging/LogEntry.java new file mode 100644 index 00000000..af88f0bf --- /dev/null +++ b/t-engine-base/src/main/java/org/alfresco/transform/base/logging/LogEntry.java @@ -0,0 +1,285 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.base.logging; + +import static java.lang.Math.max; +import static org.springframework.http.HttpStatus.OK; + +import java.text.SimpleDateFormat; +import java.util.Collection; +import java.util.Date; +import java.util.Deque; +import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.atomic.AtomicInteger; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Provides setter and getter methods to allow the current Thread to set various log properties and for these + * values to be retrieved. The {@link #complete()} method should be called at the end of a request to flush the + * current entry to an internal log Collection of the latest entries. The {@link #getLog()} method is used to obtain + * access to this collection. + */ +public final class LogEntry +{ + private static final Logger logger = LoggerFactory.getLogger(LogEntry.class); + // TODO allow ProbeTestTransform to find out if there are any transforms running longer than the max time. + + private static final AtomicInteger count = new AtomicInteger(0); + private static final Deque log = new ConcurrentLinkedDeque<>(); + private static final int MAX_LOG_SIZE = 10; + private static final SimpleDateFormat HH_MM_SS = new SimpleDateFormat("HH:mm:ss"); + + private static final ThreadLocal currentLogEntry = ThreadLocal.withInitial(() -> { + LogEntry logEntry = new LogEntry(); + if (log.size() >= MAX_LOG_SIZE) + { + log.removeLast(); + } + log.addFirst(logEntry); + return logEntry; + }); + + private final int id = count.incrementAndGet(); + private final long start = System.currentTimeMillis(); + private int statusCode; + + private long durationStreamIn; + private long durationTransform = -1; + private long durationStreamOut = -1; + private String source; + private long sourceSize; + private String target; + private long targetSize = -1; + private String options; + private String message; + + @Override + public String toString() + { + StringBuilder sb = new StringBuilder(); + append(sb, Integer.toString(getId())); + append(sb, HH_MM_SS.format(getDate())); + append(sb, Integer.toString(getStatusCode())); + append(sb, getDuration()); + append(sb, getSource()); + append(sb, getSourceSize()); + append(sb, getTarget()); + append(sb, getTargetSize()); + append(sb, getOptions()); + sb.append(getMessage()); + return sb.toString(); + } + + private void append(StringBuilder sb, String value) + { + if (value != null && !value.isEmpty() && !"0bytes".equals(value)) + { + sb.append(value); + sb.append(' '); + } + } + + public static Collection getLog() + { + return log; + } + + public static void start() + { + currentLogEntry.get(); + } + + public static void setSource(String source, long sourceSize) + { + LogEntry logEntry = currentLogEntry.get(); + logEntry.source = getExtension(source); + logEntry.sourceSize = sourceSize; + logEntry.durationStreamIn = System.currentTimeMillis() - logEntry.start; + } + + public static void setTarget(String target) + { + currentLogEntry.get().target = getExtension(target); + } + + private static String getExtension(String filename) + { + int i = filename.lastIndexOf('.'); + if (i != -1) + { + filename = filename.substring(i + 1); + } + return filename; + } + + public static void setTargetSize(long targetSize) + { + currentLogEntry.get().targetSize = targetSize; + } + + public static void setOptions(String options) + { + currentLogEntry.get().options = options; + } + + public static long setStatusCodeAndMessage(int statusCode, String message) + { + LogEntry logEntry = currentLogEntry.get(); + logEntry.statusCode = statusCode; + logEntry.message = message; + logEntry.durationTransform = System.currentTimeMillis() - logEntry.start - logEntry.durationStreamIn; + + return logEntry.durationTransform; + } + + public static void complete() + { + LogEntry logEntry = currentLogEntry.get(); + if (logEntry.statusCode == OK.value()) + { + logEntry.durationStreamOut = System.currentTimeMillis() - logEntry.start - + logEntry.durationStreamIn - max(logEntry.durationTransform, 0); + } + currentLogEntry.remove(); + + if (logger.isDebugEnabled()) + { + logger.debug(logEntry.toString()); + } + } + + public int getId() + { + return id; + } + + public Date getDate() + { + return new Date(start); + } + + public int getStatusCode() + { + return statusCode; + } + + public String getDuration() + { + long duration = durationStreamIn + max(durationTransform, 0) + max(durationStreamOut, 0); + return duration <= 5 + ? "" + : time(duration) + + " (" + + (time(durationStreamIn) + ' ' + + time(durationTransform) + ' ' + + time(durationStreamOut)).trim() + + ")"; + } + + public String getSource() + { + return source; + } + + public String getSourceSize() + { + return size(sourceSize); + } + + public String getTarget() + { + return target; + } + + public String getTargetSize() + { + return size(targetSize); + } + + public String getOptions() + { + return options; + } + + public String getMessage() + { + return message; + } + + private String time(long ms) + { + return ms == -1 ? "" : size(ms, "1ms", + new String[]{"ms", "s", "min", "hr"}, + new long[]{1000, 60 * 1000, 60 * 60 * 1000, Long.MAX_VALUE}); + } + + private String size(long size) + { + // TODO fix numeric overflow in TB expression + return size == -1 ? "" : size(size, "1 byte", + new String[]{"bytes", " KB", " MB", " GB", " TB"}, + new long[]{1024, 1024 * 1024, 1024 * 1024 * 1024, 1024 * 1024 * 1024 * 1024, Long.MAX_VALUE}); + } + + private String size(long size, String singleValue, String[] units, long[] dividers) + { + if (size == 1) + { + return singleValue; + } + long divider = 1; + for (int i = 0; i < units.length - 1; i++) + { + long nextDivider = dividers[i]; + if (size < nextDivider) + { + return unitFormat(size, divider, units[i]); + } + divider = nextDivider; + } + return unitFormat(size, divider, units[units.length - 1]); + } + + private String unitFormat(long size, long divider, String unit) + { + size = size * 10 / divider; + int decimalPoint = (int) size % 10; + + StringBuilder sb = new StringBuilder(); + sb.append(size / 10); + if (decimalPoint != 0) + { + sb.append("."); + sb.append(decimalPoint); + } + sb.append(unit); + + return sb.toString(); + } +} diff --git a/t-engine-base/src/main/java/org/alfresco/transform/base/logging/StandardMessages.java b/t-engine-base/src/main/java/org/alfresco/transform/base/logging/StandardMessages.java new file mode 100644 index 00000000..1eb6bc79 --- /dev/null +++ b/t-engine-base/src/main/java/org/alfresco/transform/base/logging/StandardMessages.java @@ -0,0 +1,35 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.base.logging; + +public interface StandardMessages +{ + String LICENCE = + "If the Alfresco software was purchased under a paid Alfresco license, the terms of the paid license agreement \n" + + "will prevail. Otherwise, the software is provided under terms of the GNU LGPL v3 license. \n" + + "See the license at http://www.gnu.org/licenses/lgpl-3.0.txt. or in /LICENSE.txt \n\n"; +} diff --git a/t-engine-base/src/main/java/org/alfresco/transform/base/messaging/MessagingConfig.java b/t-engine-base/src/main/java/org/alfresco/transform/base/messaging/MessagingConfig.java new file mode 100644 index 00000000..4f386e1f --- /dev/null +++ b/t-engine-base/src/main/java/org/alfresco/transform/base/messaging/MessagingConfig.java @@ -0,0 +1,108 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.base.messaging; + +import javax.jms.ConnectionFactory; +import javax.jms.Queue; + +import org.alfresco.transform.messages.TransformRequestValidator; +import org.apache.activemq.command.ActiveMQQueue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.jms.annotation.JmsListenerConfigurer; +import org.springframework.jms.config.DefaultJmsListenerContainerFactory; +import org.springframework.jms.config.JmsListenerEndpointRegistrar; +import org.springframework.jms.connection.JmsTransactionManager; +import org.springframework.lang.NonNull; +import org.springframework.messaging.handler.annotation.support.DefaultMessageHandlerMethodFactory; +import org.springframework.transaction.PlatformTransactionManager; + +/** + * JMS and messaging configuration for the T-Engines. Contains the basic config in order to have the + * T-Engine able to read from queues and send a reply back. + * + * @author Lucian Tuca + * created on 18/12/2018 + */ +@Configuration +@ConditionalOnProperty(name = "activemq.url") +public class MessagingConfig implements JmsListenerConfigurer +{ + private static final Logger logger = LoggerFactory.getLogger(MessagingConfig.class); + + @Override + public void configureJmsListeners(@NonNull JmsListenerEndpointRegistrar registrar) + { + registrar.setMessageHandlerMethodFactory(methodFactory()); + } + + @Bean + @ConditionalOnProperty(name = "activemq.url") + public DefaultMessageHandlerMethodFactory methodFactory() + { + DefaultMessageHandlerMethodFactory factory = new DefaultMessageHandlerMethodFactory(); + factory.setValidator(new TransformRequestValidator()); + return factory; + } + + @Bean + @ConditionalOnProperty(name = "activemq.url") + public DefaultJmsListenerContainerFactory jmsListenerContainerFactory( + final ConnectionFactory connectionFactory, + final TransformMessageConverter transformMessageConverter) + { + final DefaultJmsListenerContainerFactory factory = new DefaultJmsListenerContainerFactory(); + factory.setConnectionFactory(connectionFactory); + factory.setMessageConverter(transformMessageConverter); + factory.setErrorHandler(t -> logger.error("JMS error: " + t.getMessage(), t)); + factory.setTransactionManager(transactionManager(connectionFactory)); + return factory; + } + + @Bean + @ConditionalOnProperty(name = "activemq.url") + public PlatformTransactionManager transactionManager(final ConnectionFactory connectionFactory) + { + final JmsTransactionManager transactionManager = new JmsTransactionManager(); + transactionManager.setConnectionFactory(connectionFactory); + return transactionManager; + } + + @Bean + @ConditionalOnProperty(name = "activemq.url") + public Queue engineRequestQueue( + @Value("${queue.engineRequestQueue}") String engineRequestQueueValue) + { + return new ActiveMQQueue(engineRequestQueueValue); + } +} + + diff --git a/t-engine-base/src/main/java/org/alfresco/transform/base/messaging/MessagingInfo.java b/t-engine-base/src/main/java/org/alfresco/transform/base/messaging/MessagingInfo.java new file mode 100644 index 00000000..eb353604 --- /dev/null +++ b/t-engine-base/src/main/java/org/alfresco/transform/base/messaging/MessagingInfo.java @@ -0,0 +1,67 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.base.messaging; + +import javax.annotation.PostConstruct; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; + +/** + * Prints JMS status information at application startup. + * + * @author Cezar Leahu + */ +@Configuration +public class MessagingInfo +{ + private static final Logger logger = LoggerFactory.getLogger(MessagingInfo.class); + + @Value("${activemq.url:}") + private String activemqUrl; + + @PostConstruct + public void init() + { + // For backwards-compatibility, we continue to rely on setting ACTIVEMQ_URL environment variable (see application.yaml) + // The MessagingConfig class uses on ConditionalOnProperty (ie. activemq.url is set and not false) + + // Note: as per application.yaml the broker url is appended with "?jms.watchTopicAdvisories=false". If this needs to be fully + // overridden then it would require explicitly setting both "spring.activemq.broker-url" *and* "activemq.url" (latter to non-false value). + + if ((activemqUrl != null) && (! activemqUrl.equals("false"))) + { + logger.info("JMS client is ENABLED - ACTIVEMQ_URL ='{}'", activemqUrl); + } + else + { + logger.info("JMS client is DISABLED - ACTIVEMQ_URL is not set"); + } + } +} diff --git a/t-engine-base/src/main/java/org/alfresco/transform/base/messaging/TransformMessageConverter.java b/t-engine-base/src/main/java/org/alfresco/transform/base/messaging/TransformMessageConverter.java new file mode 100644 index 00000000..40f4113d --- /dev/null +++ b/t-engine-base/src/main/java/org/alfresco/transform/base/messaging/TransformMessageConverter.java @@ -0,0 +1,102 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ + +package org.alfresco.transform.base.messaging; + +import javax.jms.JMSException; +import javax.jms.Message; +import javax.jms.Session; + +import org.alfresco.transform.client.model.TransformReply; +import org.alfresco.transform.client.model.TransformRequest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.jms.support.converter.MappingJackson2MessageConverter; +import org.springframework.jms.support.converter.MessageConversionException; +import org.springframework.jms.support.converter.MessageConverter; +import org.springframework.jms.support.converter.MessageType; +import org.springframework.lang.NonNull; +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.type.TypeFactory; +import com.google.common.collect.ImmutableMap; + +/** + * TODO: Duplicated from the Router + * Custom wrapper over MappingJackson2MessageConverter for T-Request/T-Reply objects. + * + * @author Cezar Leahu + */ +@Service +public class TransformMessageConverter implements MessageConverter +{ + private static final Logger logger = LoggerFactory.getLogger(TransformMessageConverter.class); + + private static final MappingJackson2MessageConverter converter; + private static final JavaType TRANSFORM_REQUEST_TYPE = + TypeFactory.defaultInstance().constructType(TransformRequest.class); + + static + { + converter = new MappingJackson2MessageConverter() + { + @Override + @NonNull + protected JavaType getJavaTypeForMessage(final Message message) throws JMSException + { + if (message.getStringProperty("_type") == null) + { + return TRANSFORM_REQUEST_TYPE; + } + return super.getJavaTypeForMessage(message); + } + }; + converter.setTargetType(MessageType.BYTES); + converter.setTypeIdPropertyName("_type"); + converter.setTypeIdMappings(ImmutableMap.of( + TransformRequest.class.getName(), TransformRequest.class, + TransformReply.class.getName(), TransformReply.class) + ); + } + + @Override + @NonNull + public Message toMessage( + @NonNull final Object object, + @NonNull final Session session) throws JMSException, MessageConversionException + { + return converter.toMessage(object, session); + } + + @Override + @NonNull + public Object fromMessage(@NonNull final Message message) throws JMSException + { + return converter.fromMessage(message); + } +} diff --git a/t-engine-base/src/main/java/org/alfresco/transform/base/messaging/TransformReplySender.java b/t-engine-base/src/main/java/org/alfresco/transform/base/messaging/TransformReplySender.java new file mode 100644 index 00000000..d14c1660 --- /dev/null +++ b/t-engine-base/src/main/java/org/alfresco/transform/base/messaging/TransformReplySender.java @@ -0,0 +1,77 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.base.messaging; + +import javax.jms.Destination; + +import org.alfresco.transform.client.model.TransformReply; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jms.core.JmsTemplate; +import org.springframework.stereotype.Component; + +/** + * TODO: Duplicated from the Router + * TransformReplySender Bean + *

+ * JMS message sender/publisher + * + * @author Cezar Leahu + */ +@Component +public class TransformReplySender +{ + private static final Logger logger = LoggerFactory.getLogger(TransformReplySender.class); + + @Autowired + private JmsTemplate jmsTemplate; + + public void send(final Destination destination, final TransformReply reply) + { + send(destination, reply, reply.getRequestId()); + } + + public void send(final Destination destination, final TransformReply reply, + final String correlationId) + { + try + { + //jmsTemplate.setSessionTransacted(true); // do we need this? + jmsTemplate.convertAndSend(destination, reply, m -> { + m.setJMSCorrelationID(correlationId); + return m; + }); + logger.trace("Sent: {} - with correlation ID {}", reply, correlationId); + } + catch (Exception e) + { + logger.error( + "Failed to send T-Reply " + reply + " - for correlation ID " + correlationId, e); + } + } +} diff --git a/t-engine-base/src/main/java/org/alfresco/transform/base/metadataExtractors/AbstractMetadataExtractor.java b/t-engine-base/src/main/java/org/alfresco/transform/base/metadataExtractors/AbstractMetadataExtractor.java new file mode 100644 index 00000000..c1bb9817 --- /dev/null +++ b/t-engine-base/src/main/java/org/alfresco/transform/base/metadataExtractors/AbstractMetadataExtractor.java @@ -0,0 +1,599 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005-2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.base.metadataExtractors; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.alfresco.transform.common.TransformException; +import org.slf4j.Logger; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.Serializable; +import java.lang.reflect.Array; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.StringTokenizer; +import java.util.TreeMap; + +/** + * Helper methods for metadata extract and embed. + *

+ * Much of the code is based on AbstractMappingMetadataExtracter from the + * content repository. The code has been simplified to only set up mapping one way. + *

+ * If a transform specifies that it can convert from {@code ""} to {@code "alfresco-metadata-extract"} + * (specified in the {@code engine_config.json}), it is indicating that it can extract metadata from {@code }. + * + * The transform results in a Map of extracted properties encoded as json being returned to the content repository. + *

    + *
  • The content repository will use a transform in preference to any metadata extractors it might have defined + * locally for the same MIMETYPE.
    • + *
  • The T-Engine's Controller class will call a method in a class that extends {@link AbstractMetadataExtractor} + * based on the source and target mediatypes in the normal way.
    • + *
  • The method extracts ALL available metadata is extracted from the document and then calls + * {@link #mapMetadataAndWrite(File, Map, Map)}.
    • + *
  • Selected values from the available metadata are mapped into content repository property names and values, + * depending on what is defined in a {@code "_metadata_extract.properties"} file.
    • + *
  • The selected values are set back to the content repository as a JSON representation of a Map, where the values + * are applied to the source node.
    • + *
+ * To support the same functionality as metadata extractors configured inside the content repository, + * extra key value pairs may be returned from {@link #extractMetadata}. These are: + *
    + *
  • {@code "sys:overwritePolicy"} which can specify the + * {@code org.alfresco.repo.content.metadata.MetadataExtracter.OverwritePolicy} name. Defaults to "PRAGMATIC".
    • + *
  • {@code "sys:enableStringTagging"} if {@code "true"} finds or creates tags for each string mapped to + * {@code cm:taggable}. Defaults to {@code "false"} to ignore mapping strings to tags.
    • + *
  • {@code "sys:carryAspectProperties"}
    • + *
  • {@code "sys:stringTaggingSeparators"}
    • + *
+ * + * If a transform specifies that it can convert from {@code ""} to {@code "alfresco-metadata-embed"}, it is + * indicating that it can embed metadata in {@code }. + * + * The transform results in a new version of supplied source file that contains the metadata supplied in the transform + * options. + * + * @author Jesper Steen Møller + * @author Derek Hulley + * @author adavis + */ +public abstract class AbstractMetadataExtractor +{ + private static final String EXTRACT = "extract"; + private static final String EMBED = "embed"; + private static final String METADATA = "metadata"; + private static final String EXTRACT_MAPPING = "extractMapping"; + + private static final String NAMESPACE_PROPERTY_PREFIX = "namespace.prefix."; + private static final char NAMESPACE_PREFIX = ':'; + private static final char NAMESPACE_BEGIN = '{'; + private static final char NAMESPACE_END = '}'; + + private static final List SYS_PROPERTIES = Arrays.asList( + "sys:overwritePolicy", + "sys:enableStringTagging", + "sys:carryAspectProperties", + "sys:stringTaggingSeparators"); + + private static final ObjectMapper jsonObjectMapper = new ObjectMapper(); + + protected final Logger logger; + private Map> defaultExtractMapping; + private ThreadLocal>> extractMapping = new ThreadLocal<>(); + private Map> embedMapping; + + public AbstractMetadataExtractor(Logger logger) + { + this.logger = logger; + defaultExtractMapping = Collections.emptyMap(); + embedMapping = Collections.emptyMap(); + try + { + defaultExtractMapping = buildExtractMapping(); + embedMapping = buildEmbedMapping(); + } + catch (Exception e) + { + logger.error("Failed to read config", e); + } + } + + public abstract Map extractMetadata(String sourceMimetype, Map transformOptions, + File sourceFile) throws Exception; + + public void embedMetadata(String sourceMimetype, String targetMimetype, Map transformOptions, + File sourceFile, File targetFile) throws Exception + { + // Default nothing, as embedding is not supported in most cases + } + + protected Map getMetadata(Map transformOptions) + { + String metadataAsJson = transformOptions.get(METADATA); + if (metadataAsJson == null) + { + throw new IllegalArgumentException("No metadata in embed request"); + } + + try + { + TypeReference> typeRef = new TypeReference<>() {}; + HashMap systemProperties = jsonObjectMapper.readValue(metadataAsJson, typeRef); + Map rawProperties = mapSystemToRaw(systemProperties); + return rawProperties; + } + catch (JsonProcessingException e) + { + throw new IllegalArgumentException("Failed to read metadata from request", e); + } + } + + private Map mapSystemToRaw(Map systemMetadata) + { + Map metadataProperties = new HashMap<>(systemMetadata.size() * 2 + 1); + for (Map.Entry entry : systemMetadata.entrySet()) + { + String modelProperty = entry.getKey(); + // Check if there is a mapping for this + if (!embedMapping.containsKey(modelProperty)) + { + // No mapping - ignore + continue; + } + Serializable documentValue = entry.getValue(); + Set metadataKeys = embedMapping.get(modelProperty); + for (String metadataKey : metadataKeys) + { + metadataProperties.put(metadataKey, documentValue); + } + } + // Done + if (logger.isDebugEnabled()) + { + logger.debug( + "Converted system model values to metadata values: \n" + + " System Properties: " + systemMetadata + "\n" + + " Metadata Properties: " + metadataProperties); + } + return metadataProperties; + } + + protected Map> getExtractMapping() + { + return Collections.unmodifiableMap(extractMapping.get()); + } + + public Map> getEmbedMapping() + { + return Collections.unmodifiableMap(embedMapping); + } + + /** + * Based on AbstractMappingMetadataExtracter#getDefaultMapping. + * + * This method provides a mapping of where to store the values extracted from the documents. The list of + * properties need not include all metadata values extracted from the document. This mapping should be + * defined in a file based on the class name: {@code "_metadata_extract.properties"} + * @return Returns a static mapping. It may not be null. + */ + private Map> buildExtractMapping() + { + String filename = getPropertiesFilename(EXTRACT); + Properties properties = readProperties(filename); + if (properties == null) + { + logger.error("Failed to read "+filename); + } + + Map namespacesByPrefix = getNamespaces(properties); + return buildExtractMapping(properties, namespacesByPrefix); + } + + private Map> buildExtractMapping(Properties properties, Map namespacesByPrefix) + { + // Create the mapping + Map> convertedMapping = new HashMap<>(17); + for (Map.Entry entry : properties.entrySet()) + { + String documentProperty = (String) entry.getKey(); + String qnamesStr = (String) entry.getValue(); + if (documentProperty.startsWith(NAMESPACE_PROPERTY_PREFIX)) + { + continue; + } + // Create the entry + Set qnames = new HashSet<>(3); + convertedMapping.put(documentProperty, qnames); + // The to value can be a list of QNames + StringTokenizer tokenizer = new StringTokenizer(qnamesStr, ","); + while (tokenizer.hasMoreTokens()) + { + String qnameStr = tokenizer.nextToken().trim(); + qnameStr = getQNameString(namespacesByPrefix, entry, qnameStr, EXTRACT); + qnames.add(qnameStr); + } + if (logger.isTraceEnabled()) + { + logger.trace("Added mapping from " + documentProperty + " to " + qnames); + } + } + return convertedMapping; + } + + /** + * Based on AbstractMappingMetadataExtracter#getDefaultEmbedMapping. + * + * This method provides a mapping of model properties that should be embedded in the content. The list of + * properties need not include all properties. This mapping should be defined in a file based on the class + * name: {@code "_metadata_embed.properties"} + *

+ * If no {@code "_metadata_embed.properties"} file is found, a reverse of the + * {@code "_metadata_extract.properties"} will be assumed. A last win approach will be used for handling + * duplicates. + * @return Returns a static mapping. It may not be null. + */ + private Map> buildEmbedMapping() + { + String filename = getPropertiesFilename(EMBED); + Properties properties = readProperties(filename); + + Map> embedMapping; + if (properties != null) + { + Map namespacesByPrefix = getNamespaces(properties); + embedMapping = buildEmbedMapping(properties, namespacesByPrefix); + } + else + { + if (logger.isDebugEnabled()) + { + logger.debug("No " + filename + ", assuming reverse of extract mapping"); + } + embedMapping = buildEmbedMappingByReversingExtract(); + } + return embedMapping; + } + + private Map> buildEmbedMapping(Properties properties, Map namespacesByPrefix) + { + Map> convertedMapping = new HashMap<>(17); + for (Map.Entry entry : properties.entrySet()) + { + String modelProperty = (String) entry.getKey(); + String metadataKeysString = (String) entry.getValue(); + if (modelProperty.startsWith(NAMESPACE_PROPERTY_PREFIX)) + { + continue; + } + + modelProperty = getQNameString(namespacesByPrefix, entry, modelProperty, EMBED); + String[] metadataKeysArray = metadataKeysString.split(","); + Set metadataKeys = new HashSet(metadataKeysArray.length); + for (String metadataKey : metadataKeysArray) { + metadataKeys.add(metadataKey.trim()); + } + // Create the entry + convertedMapping.put(modelProperty, metadataKeys); + if (logger.isTraceEnabled()) + { + logger.trace("Added mapping from " + modelProperty + " to " + metadataKeysString); + } + } + return convertedMapping; + } + + private Map> buildEmbedMappingByReversingExtract() + { + Map> extractMapping = buildExtractMapping(); + Map> embedMapping; + embedMapping = new HashMap<>(extractMapping.size()); + for (String metadataKey : extractMapping.keySet()) + { + if (extractMapping.get(metadataKey) != null && extractMapping.get(metadataKey).size() > 0) + { + String modelProperty = extractMapping.get(metadataKey).iterator().next(); + Set metadataKeys = embedMapping.get(modelProperty); + if (metadataKeys == null) + { + metadataKeys = new HashSet(1); + embedMapping.put(modelProperty, metadataKeys); + } + metadataKeys.add(metadataKey); + if (logger.isTraceEnabled()) + { + logger.trace("Added mapping from " + modelProperty + " to " + metadataKeys.toString()); + } + } + } + return embedMapping; + } + + private String getPropertiesFilename(String suffix) + { + String className = this.getClass().getName(); + String shortClassName = className.split("\\.")[className.split("\\.").length - 1]; + shortClassName = shortClassName.replace('$', '-'); + + return shortClassName + "_metadata_" + suffix + ".properties"; + } + + private Properties readProperties(String filename) + { + Properties properties = null; + try + { + InputStream inputStream = AbstractMetadataExtractor.class.getClassLoader().getResourceAsStream(filename); + if (inputStream != null) + { + properties = new Properties(); + properties.load(inputStream); + } + } + catch (IOException ignore) + { + } + return properties; + } + + private Map getNamespaces(Properties properties) + { + Map namespacesByPrefix = new HashMap(5); + for (Map.Entry entry : properties.entrySet()) + { + String propertyName = (String) entry.getKey(); + if (propertyName.startsWith(NAMESPACE_PROPERTY_PREFIX)) + { + String prefix = propertyName.substring(17); + String namespace = (String) entry.getValue(); + namespacesByPrefix.put(prefix, namespace); + } + } + return namespacesByPrefix; + } + + private String getQNameString(Map namespacesByPrefix, Map.Entry entry, String qnameStr, String type) + { + // Check if we need to resolve a namespace reference + int index = qnameStr.indexOf(NAMESPACE_PREFIX); + if (index > -1 && qnameStr.charAt(0) != NAMESPACE_BEGIN) + { + String prefix = qnameStr.substring(0, index); + String suffix = qnameStr.substring(index + 1); + // It is prefixed + String uri = namespacesByPrefix.get(prefix); + if (uri == null) + { + throw new IllegalArgumentException("No prefix mapping for " + type + " property mapping: \n" + + " Extractor: " + this + "\n" + + " Mapping: " + entry); + } + qnameStr = NAMESPACE_BEGIN + uri + NAMESPACE_END + suffix; + } + return qnameStr; + } + + /** + * Adds a value to the map, conserving null values. Values are converted to null if: + *

    + *
  • it is an empty string value after trimming
    • + *
  • it is an empty collection
    • + *
  • it is an empty array
    • + *
+ * String values are trimmed before being put into the map. + * Otherwise, it is up to the extracter to ensure that the value is a Serializable. + * It is not appropriate to implicitly convert values in order to make them Serializable + * - the best conversion method will depend on the value's specific meaning. + * + * @param key the destination key + * @param value the serializable value + * @param destination the map to put values into + * @return Returns true if set, otherwise false + */ + // Copied from the content repository's AbstractMappingMetadataExtracter. + protected boolean putRawValue(String key, Serializable value, Map destination) + { + if (value == null) + { + // Just keep this + } + else if (value instanceof String) + { + String valueStr = ((String) value).trim(); + if (valueStr.length() == 0) + { + value = null; + } + else + { + if (valueStr.indexOf("\u0000") != -1) + { + valueStr = valueStr.replaceAll("\u0000", ""); + } + // Keep the trimmed value + value = valueStr; + } + } + else if (value instanceof Collection) + { + Collection valueCollection = (Collection) value; + if (valueCollection.isEmpty()) + { + value = null; + } + } + else if (value.getClass().isArray()) + { + if (Array.getLength(value) == 0) + { + value = null; + } + } + // It passed all the tests + destination.put(key, value); + return true; + } + + public void extractMetadata(String sourceMimetype, Map transformOptions, + String sourceEncoding, InputStream inputStream, + String targetEncoding, OutputStream outputStream) throws Exception + { + // TODO + throw new TransformException(500, "TODO extractMetadata"); + } + + /** + * The {@code transformOptions} may contain a replacement set of mappings. These will be used in place of the + * default mappings from read from file if supplied. + */ + public void extractMetadata(String sourceMimetype, Map transformOptions, File sourceFile, + File targetFile) throws Exception + { + Map> mapping = getExtractMappingFromOptions(transformOptions, defaultExtractMapping); + + // Use a ThreadLocal to avoid changing method signatures of methods that currently call getExtractMapping. + try + { + extractMapping.set(mapping); + Map metadata = extractMetadata(sourceMimetype, transformOptions, sourceFile); + mapMetadataAndWrite(targetFile, metadata, mapping); + + } + finally + { + extractMapping.set(null); + } + } + + private Map> getExtractMappingFromOptions(Map transformOptions, Map> defaultExtractMapping) + { + String extractMappingOption = transformOptions.get(EXTRACT_MAPPING); + if (extractMappingOption != null) + { + try + { + TypeReference>> typeRef = new TypeReference<>() {}; + return jsonObjectMapper.readValue(extractMappingOption, typeRef); + } + catch (JsonProcessingException e) + { + throw new IllegalArgumentException("Failed to read "+ EXTRACT_MAPPING +" from request", e); + } + } + return defaultExtractMapping; + } + + /** + * @deprecated use {@link #extractMetadata(String, Map, File, File)} rather than calling this method. + * By default call the overloaded method with the default {@code extractMapping}. + */ + @Deprecated + public void mapMetadataAndWrite(File targetFile, Map metadata) throws IOException + { + mapMetadataAndWrite(targetFile, metadata, defaultExtractMapping); + } + + public void mapMetadataAndWrite(File targetFile, Map metadata, + Map> extractMapping) throws IOException + { + if (logger.isDebugEnabled()) + { + logger.debug("Raw metadata:"); + metadata.forEach((k,v) -> logger.debug(" "+k+"="+v)); + } + + metadata = mapRawToSystem(metadata, extractMapping); + writeMetadata(targetFile, metadata); + } + + /** + * Based on AbstractMappingMetadataExtracter#mapRawToSystem. + * + * @param rawMetadata Metadata keyed by document properties + * @param extractMapping Mapping between document ans system properties + * @return Returns the metadata keyed by the system properties + */ + private Map mapRawToSystem(Map rawMetadata, + Map> extractMapping) + { + boolean debugEnabled = logger.isDebugEnabled(); + if (debugEnabled) + { + logger.debug("Returned metadata:"); + } + Map systemProperties = new HashMap(rawMetadata.size() * 2 + 1); + for (Map.Entry entry : rawMetadata.entrySet()) + { + String documentKey = entry.getKey(); + Serializable documentValue = entry.getValue(); + if (SYS_PROPERTIES.contains(documentKey)) + { + systemProperties.put(documentKey, documentValue); + if (debugEnabled) + { + logger.debug(" " + documentKey + "=" + documentValue); + } + continue; + } + // Check if there is a mapping for this + if (!extractMapping.containsKey(documentKey)) + { + // No mapping - ignore + continue; + } + + Set systemQNames = extractMapping.get(documentKey); + for (String systemQName : systemQNames) + { + if (debugEnabled) + { + logger.debug(" "+systemQName+"="+documentValue+" ("+documentKey+")"); + } + systemProperties.put(systemQName, documentValue); + } + } + return new TreeMap(systemProperties); + } + + private void writeMetadata(File targetFile, Map results) + throws IOException + { + jsonObjectMapper.writeValue(targetFile, results); + } +} diff --git a/t-engine-base/src/main/java/org/alfresco/transform/base/model/FileRefEntity.java b/t-engine-base/src/main/java/org/alfresco/transform/base/model/FileRefEntity.java new file mode 100644 index 00000000..22aa2bc9 --- /dev/null +++ b/t-engine-base/src/main/java/org/alfresco/transform/base/model/FileRefEntity.java @@ -0,0 +1,77 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.base.model; + +import java.util.Objects; + +/** + * TODO: Copied from org.alfresco.store.entity (alfresco-shared-file-store). To be discussed + * + * POJO that represents content reference ({@link java.util.UUID}) + */ +public class FileRefEntity +{ + private String fileRef; + + public FileRefEntity() {} + + public FileRefEntity(String fileRef) + { + this.fileRef = fileRef; + } + + public void setFileRef(String fileRef) + { + this.fileRef = fileRef; + } + + public String getFileRef() + { + return fileRef; + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + FileRefEntity that = (FileRefEntity) o; + return Objects.equals(fileRef, that.fileRef); + } + + @Override + public int hashCode() + { + return Objects.hash(fileRef); + } + + @Override + public String toString() + { + return fileRef; + } +} diff --git a/t-engine-base/src/main/java/org/alfresco/transform/base/model/FileRefResponse.java b/t-engine-base/src/main/java/org/alfresco/transform/base/model/FileRefResponse.java new file mode 100644 index 00000000..5d80d441 --- /dev/null +++ b/t-engine-base/src/main/java/org/alfresco/transform/base/model/FileRefResponse.java @@ -0,0 +1,54 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.base.model; + +/** + * TODO: Copied from org.alfresco.store.entity (alfresco-shared-file-store). To be discussed + * + * POJO that describes the ContentRefEntry response, contains {@link FileRefEntity} according to API spec + */ +public class FileRefResponse +{ + private FileRefEntity entry; + + public FileRefResponse() {} + + public FileRefResponse(FileRefEntity entry) + { + this.entry = entry; + } + + public FileRefEntity getEntry() + { + return entry; + } + + public void setEntry(FileRefEntity entry) + { + this.entry = entry; + } +} diff --git a/t-engine-base/src/main/java/org/alfresco/transform/base/probes/ProbeTestTransform.java b/t-engine-base/src/main/java/org/alfresco/transform/base/probes/ProbeTestTransform.java new file mode 100644 index 00000000..430b96eb --- /dev/null +++ b/t-engine-base/src/main/java/org/alfresco/transform/base/probes/ProbeTestTransform.java @@ -0,0 +1,391 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.base.probes; + +import static org.alfresco.transform.base.fs.FileManager.SOURCE_FILE; +import static org.alfresco.transform.base.fs.FileManager.TARGET_FILE; +import static org.alfresco.transform.base.fs.FileManager.TempFileProvider.createTempFile; +import static org.springframework.http.HttpStatus.BAD_REQUEST; +import static org.springframework.http.HttpStatus.INSUFFICIENT_STORAGE; +import static org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR; +import static org.springframework.http.HttpStatus.OK; +import static org.springframework.http.HttpStatus.TOO_MANY_REQUESTS; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +import javax.servlet.http.HttpServletRequest; + +import org.alfresco.transform.base.TransformController; +import org.alfresco.transform.common.TransformException; +import org.alfresco.transform.registry.TransformServiceRegistry; +import org.alfresco.transform.base.logging.LogEntry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * Provides test transformations and the logic used by k8 liveness and readiness probes. + * + *

K8s probes: A readiness probe indicates if the pod should accept request. It does not indicate that a pod is + * ready after startup. The liveness probe indicates when to kill the pod. Both probes are called throughout the + * lifetime of the pod and a liveness probes can take place before a readiness probe. The k8s + * initialDelaySeconds field is not fully honoured as it is multiplied by a random number, so is + * actually a maximum initial delay in seconds, but could be 0.

+ * + *

Live and readiness probes do test transforms. The first 6 requests result in a transformation of a small test + * file. The average time and size is remembered, but excludes the first one as it is normally slower. This is + * used in future requests to discover if transformations are becoming slower or unexpectedly change size.

+ * + *

If a transform longer than a maximum time, a maximum number of transforms have been performed, a test transform is + * an unexpected size or a test transform takes an unexpected time, then a non 200 status code is returned resulting in + * k8s terminating the pod. These are controlled by:

+ *
    + *
  • expectedLength the expected length of the target file after a test transform
    • + *
  • plusOrMinus allows for variation in the transformed size - generally caused by dates
    • + *
  • livenessPercent allows for variation in transform time. Up to 2 and a half times is not + * unreasonable under load
    • + *
  • maxTransforms the maximum number of transforms (not just test ones) before a restart is triggered
    • + *
  • maxTransformSeconds a maximum time any transform (not just test ones) is allowed to take before + * a restart is triggered.
    • + *
  • livenessTransformPeriodSeconds The number of seconds between test transforms done for live probes
    • + *
+ */ +public class ProbeTestTransform +{ + private final Logger logger = LoggerFactory.getLogger(ProbeTestTransform.class); + + @Autowired + private TransformServiceRegistry transformRegistry; + + private static final int AVERAGE_OVER_TRANSFORMS = 5; + private final String sourceFilename; + private final String targetFilename; + private final String sourceMimetype; + private final String targetMimetype; + private final Map transformOptions; + private final long minExpectedLength; + private final long maxExpectedLength; + + private int livenessPercent; + private long probeCount; + private int transCount; + private long normalTime; + private long maxTime = Long.MAX_VALUE; + private long nextTransformTime; + + private final boolean livenessTransformEnabled; + private final long livenessTransformPeriod; + private final long maxTransformCount; + private long maxTransformTime; + + private final AtomicBoolean initialised = new AtomicBoolean(false); + private final AtomicBoolean readySent = new AtomicBoolean(false); + private final AtomicLong transformCount = new AtomicLong(0); + private final AtomicBoolean die = new AtomicBoolean(false); + + public int getLivenessPercent() + { + return livenessPercent; + } + + public long getMaxTime() + { + return maxTime; + } + + public ProbeTestTransform(String sourceFilename, String targetFilename, + String sourceMimetype, String targetMimetype, Map transformOptions, + long expectedLength, long plusOrMinus, int livenessPercent, long maxTransforms, long maxTransformSeconds, + long livenessTransformPeriodSeconds) + { + this.sourceFilename = sourceFilename; + this.targetFilename = targetFilename; + this.sourceMimetype = sourceMimetype; + this.targetMimetype = targetMimetype; + this.transformOptions = transformOptions; + minExpectedLength = Math.max(0, expectedLength - plusOrMinus); + maxExpectedLength = expectedLength + plusOrMinus; + + this.livenessPercent = (int) getPositiveLongEnv("livenessPercent", livenessPercent); + maxTransformCount = getPositiveLongEnv("maxTransforms", maxTransforms); + maxTransformTime = getPositiveLongEnv("maxTransformSeconds", maxTransformSeconds) * 1000; + livenessTransformPeriod = getPositiveLongEnv("livenessTransformPeriodSeconds", + livenessTransformPeriodSeconds) * 1000; + livenessTransformEnabled = getBooleanEnvVar("livenessTransformEnabled", false); + } + + private boolean getBooleanEnvVar(final String name, final boolean defaultValue) + { + try + { + return Boolean.parseBoolean(System.getenv(name)); + } + catch (Exception ignore) + { + } + return defaultValue; + } + + private long getPositiveLongEnv(String name, long defaultValue) + { + long l = -1; + String env = System.getenv(name); + if (env != null) + { + try + { + l = Long.parseLong(env); + } + catch (NumberFormatException ignore) + { + } + } + if (l <= 0) + { + l = defaultValue; + } + logger.trace("Probe: {}={}", name, l); + return l; + } + + // We don't want to be doing test transforms every few seconds, but do want frequent live probes. + public String doTransformOrNothing(HttpServletRequest request, boolean isLiveProbe, TransformController controller) + { + // If not initialised OR it is a live probe and we are scheduled to to do a test transform. + probeCount++; + // TODO: update/fix/refactor liveness probes as part of ATS-138 + if (isLiveProbe && !livenessTransformEnabled) + { + return doNothing(true); + } + return (isLiveProbe && livenessTransformPeriod > 0 && + (transCount <= AVERAGE_OVER_TRANSFORMS || nextTransformTime < System.currentTimeMillis())) + || !initialised.get() + ? doTransform(request, isLiveProbe, controller) + : doNothing(isLiveProbe); + } + + private String doNothing(boolean isLiveProbe) + { + String probeMessage = getProbeMessage(isLiveProbe); + String message = "Success - No transform."; + LogEntry.setStatusCodeAndMessage(OK.value(), probeMessage + message); + if (!isLiveProbe && !readySent.getAndSet(true)) + { + logger.trace("{}{}", probeMessage, message); + } + return message; + } + + private String doTransform(HttpServletRequest request, boolean isLiveProbe, TransformController controller) + { + checkMaxTransformTimeAndCount(isLiveProbe); + + long start = System.currentTimeMillis(); + + if (nextTransformTime != 0) + { + do + { + nextTransformTime += livenessTransformPeriod; + } + while (nextTransformTime < start); + } + + File sourceFile = getSourceFile(request, isLiveProbe); + File targetFile = getTargetFile(request); + + String transformName = getTransformerName(sourceFile, sourceMimetype, targetMimetype, transformOptions); + controller.transformImpl(transformName, sourceMimetype, targetMimetype, transformOptions, sourceFile, targetFile); + + long time = System.currentTimeMillis() - start; + String message = "Transform " + time + "ms"; + checkTargetFile(targetFile, isLiveProbe, message); + + recordTransformTime(time); + calculateMaxTime(time, isLiveProbe); + + if (time > maxTime) + { + throw new TransformException(INTERNAL_SERVER_ERROR.value(), + getMessagePrefix(isLiveProbe) + + message + " which is more than " + livenessPercent + + "% slower than the normal value of " + normalTime + "ms"); + } + + // We don't care if the ready or live probe works out if we are 'ready' to take requests. + initialised.set(true); + + checkMaxTransformTimeAndCount(isLiveProbe); + + return getProbeMessage(isLiveProbe) + message; + } + + private String getTransformerName(final File sourceFile, final String sourceMimetype, + final String targetMimetype, final Map transformOptions) + { + final long sourceSizeInBytes = sourceFile.length(); + final String transformerName = transformRegistry.findTransformerName(sourceMimetype, + sourceSizeInBytes, targetMimetype, transformOptions, null); + if (transformerName == null) + { + throw new TransformException(BAD_REQUEST.value(), "No transforms were able to handle the request"); + } + return transformerName; + } + + private void checkMaxTransformTimeAndCount(boolean isLiveProbe) + { + if (die.get()) + { + throw new TransformException(TOO_MANY_REQUESTS.value(), + getMessagePrefix(isLiveProbe) + "Transformer requested to die. A transform took " + + "longer than " + (maxTransformTime * 1000) + " seconds"); + } + + if (maxTransformCount > 0 && transformCount.get() > maxTransformCount) + { + throw new TransformException(TOO_MANY_REQUESTS.value(), + getMessagePrefix(isLiveProbe) + "Transformer requested to die. It has performed " + + "more than " + maxTransformCount + " transformations"); + } + } + + private File getSourceFile(HttpServletRequest request, boolean isLiveProbe) + { + incrementTransformerCount(); + File sourceFile = createTempFile("source_", "_" + sourceFilename); + request.setAttribute(SOURCE_FILE, sourceFile); + try (InputStream inputStream = this.getClass().getResourceAsStream('/' + sourceFilename)) + { + Files.copy(inputStream, sourceFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + } + catch (IOException e) + { + throw new TransformException(INSUFFICIENT_STORAGE.value(), + getMessagePrefix(isLiveProbe) + "Failed to store the source file", e); + } + long length = sourceFile.length(); + LogEntry.setSource(sourceFilename, length); + return sourceFile; + } + + private File getTargetFile(HttpServletRequest request) + { + File targetFile = createTempFile("target_", "_" + targetFilename); + request.setAttribute(TARGET_FILE, targetFile); + LogEntry.setTarget(targetFilename); + return targetFile; + } + + public void recordTransformTime(long time) + { + if (maxTransformTime > 0 && time > maxTransformTime) + { + die.set(true); + } + } + + public void calculateMaxTime(long time, boolean isLiveProbe) + { + if (transCount <= AVERAGE_OVER_TRANSFORMS) + { + // Take the average of the first few transforms as the normal time. The initial transform might be slower + // so is ignored. Later ones are not included in case we have a gradual performance problem. + String message = getMessagePrefix(isLiveProbe) + "Success - Transform " + time + "ms"; + if (++transCount > 1) + { + normalTime = (normalTime * (transCount - 2) + time) / (transCount - 1); + maxTime = (normalTime * (livenessPercent + 100)) / 100; + + if ((!isLiveProbe && !readySent.getAndSet( + true)) || transCount > AVERAGE_OVER_TRANSFORMS) + { + nextTransformTime = System.currentTimeMillis() + livenessTransformPeriod; + logger.trace("{} - {}ms+{}%={}ms", message, normalTime, livenessPercent, + maxTime); + } + } + else if (!isLiveProbe && !readySent.getAndSet(true)) + { + logger.trace(message); + } + } + } + + private void checkTargetFile(File targetFile, boolean isLiveProbe, String message) + { + String probeMessage = getProbeMessage(isLiveProbe); + if (!targetFile.exists() || !targetFile.isFile()) + { + throw new TransformException(INTERNAL_SERVER_ERROR.value(), + probeMessage + "Target File \"" + targetFile.getAbsolutePath() + "\" did not exist"); + } + long length = targetFile.length(); + if (length < minExpectedLength || length > maxExpectedLength) + { + throw new TransformException(INTERNAL_SERVER_ERROR.value(), + probeMessage + "Target File \"" + targetFile.getAbsolutePath() + + "\" was the wrong size (" + length + "). Needed to be between " + + minExpectedLength + " and " + maxExpectedLength); + } + LogEntry.setTargetSize(length); + LogEntry.setStatusCodeAndMessage(OK.value(), probeMessage + "Success - " + message); + } + + private String getMessagePrefix(boolean isLiveProbe) + { + return Long.toString(probeCount) + ' ' + getProbeMessage(isLiveProbe); + } + + private String getProbeMessage(boolean isLiveProbe) + { + return (isLiveProbe ? "Live Probe: " : "Ready Probe: "); + } + + public void incrementTransformerCount() + { + transformCount.incrementAndGet(); + } + + public void setLivenessPercent(int livenessPercent) + { + this.livenessPercent = livenessPercent; + } + + public long getNormalTime() + { + return normalTime; + } +} diff --git a/t-engine-base/src/main/java/org/alfresco/transform/base/util/RequestParamMap.java b/t-engine-base/src/main/java/org/alfresco/transform/base/util/RequestParamMap.java new file mode 100644 index 00000000..598fc26d --- /dev/null +++ b/t-engine-base/src/main/java/org/alfresco/transform/base/util/RequestParamMap.java @@ -0,0 +1,41 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.base.util; + +/** + * Extends the list of transform options with historic request parameters or 'extra' parameters used in testing + * or communication in the all-in-one transformer. + */ +public interface RequestParamMap extends org.alfresco.transform.common.RequestParamMap +{ + String TRANSFORM_NAME_PARAMETER = "alfresco.transform-name-parameter"; + + String FILE = "file"; + String SOURCE_EXTENSION = "sourceExtension"; + String SOURCE_MIMETYPE = "sourceMimetype"; + String TARGET_MIMETYPE = "targetMimetype"; +} diff --git a/t-engine-base/src/main/java/org/alfresco/transform/base/util/Util.java b/t-engine-base/src/main/java/org/alfresco/transform/base/util/Util.java new file mode 100644 index 00000000..ba5a8c8e --- /dev/null +++ b/t-engine-base/src/main/java/org/alfresco/transform/base/util/Util.java @@ -0,0 +1,63 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.base.util; + +public class Util +{ + /** + * Safely converts a {@link String} to an {@link Integer} + * + * @param param String to be converted + * @return Null if param is null or converted value as {@link Integer} + */ + public static Integer stringToInteger(final String param) + { + return param == null ? null : Integer.parseInt(param); + } + + /** + * Safely converts a {@link String} to a {@link Boolean} + * + * @param param String to be converted + * @return Null if param is null or converted value as {@link Boolean} + */ + public static Boolean stringToBoolean(final String param) + { + return param == null ? null : Boolean.parseBoolean(param); + } + + /** + * Safely converts a {@link String} to a {@link Long} + * + * @param param String to be converted + * @return Null if param is null or converted value as {@link Boolean} + */ + public static Long stringToLong(final String param) + { + return param == null ? null : Long.parseLong(param); + } +} diff --git a/t-engine-base/src/main/resources/application.yaml b/t-engine-base/src/main/resources/application.yaml new file mode 100644 index 00000000..ea509310 --- /dev/null +++ b/t-engine-base/src/main/resources/application.yaml @@ -0,0 +1,61 @@ +--- +spring: + servlet: + multipart: + max-file-size: 8192MB + max-request-size: 8192MB + activemq: + broker-url: ${ACTIVEMQ_URL:nio://localhost:61616}?jms.watchTopicAdvisories=false + user: ${ACTIVEMQ_USER:admin} + password: ${ACTIVEMQ_PASSWORD:admin} + pool: + enabled: true + max-connections: 20 + jackson: + default-property-inclusion: non_empty + +activemq: + url: ${ACTIVEMQ_URL:false} + +server: + port: ${SERVER_PORT:8090} + error: + include-message: ALWAYS + +transform: + core: + version: @project.version@ + +logging: + level: + # org.alfresco.util.exec.RuntimeExec: debug + org.alfresco.transformer.LibreOfficeController: debug + org.alfresco.transformer.JodConverterSharedInstance: debug + org.alfresco.transformer.AlfrescoPdfRendererController: debug + org.alfresco.transformer.ImageMagickController: debug + org.alfresco.transformer.TikaController: debug + org.alfresco.transformer.MiscellaneousTransformersController: debug + org.alfresco.transform.common.TransformerDebug: debug + +fileStoreUrl: ${FILE_STORE_URL:http://localhost:8099/alfresco/api/-default-/private/sfs/versions/1/file} + +jms-listener: + concurrency: ${JMS_LISTENER_CONCURRENCY:1-10} + +management: + endpoints: + web: + exposure: + include: + - metrics + - prometheus + - health + metrics: + enable[http]: false + enable[logback]: false + enable[tomcat]: false + enable[jvm.classes]: false + +container: + name: ${HOSTNAME:t-engine} + diff --git a/t-engine-base/src/main/resources/engine_config.json b/t-engine-base/src/main/resources/engine_config.json new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/t-engine-base/src/main/resources/engine_config.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/t-engine-base/src/main/resources/templates/error.html b/t-engine-base/src/main/resources/templates/error.html new file mode 100644 index 00000000..54b0facd --- /dev/null +++ b/t-engine-base/src/main/resources/templates/error.html @@ -0,0 +1,22 @@ + + + +
+
+

Error Page

+
+
+

+

+
+

+
+ + + + + diff --git a/t-engine-base/src/main/resources/templates/log.html b/t-engine-base/src/main/resources/templates/log.html new file mode 100644 index 00000000..ee973306 --- /dev/null +++ b/t-engine-base/src/main/resources/templates/log.html @@ -0,0 +1,43 @@ + + + +
+

+
+ + + + + + + + + + + + + + + + + + + + + + + + + +
IdTimeStatus CodeDuration (ms)SourceTargetOptionsMessage
+
+
+ + + + + diff --git a/t-engine-base/src/test/java/org/alfresco/transform/base/AbstractHttpRequestTest.java b/t-engine-base/src/test/java/org/alfresco/transform/base/AbstractHttpRequestTest.java new file mode 100644 index 00000000..c6e8ae15 --- /dev/null +++ b/t-engine-base/src/test/java/org/alfresco/transform/base/AbstractHttpRequestTest.java @@ -0,0 +1,177 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.base; + +import static org.alfresco.transform.common.RequestParamMap.DIRECT_ACCESS_URL; +import static org.alfresco.transform.common.RequestParamMap.ENDPOINT_TRANSFORM; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.springframework.http.HttpMethod.POST; +import static org.springframework.http.MediaType.MULTIPART_FORM_DATA; +import static org.springframework.test.util.AssertionErrors.assertTrue; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.web.client.TestRestTemplate; +import org.springframework.boot.web.server.LocalServerPort; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.ResponseEntity; +import org.springframework.util.LinkedMultiValueMap; + +/** + * Super class for testing controllers with a server. Includes tests for the Controller itself. + * Note: Currently uses json rather than HTML as json is returned by this spring boot test harness. + */ +public abstract class AbstractHttpRequestTest +{ + @LocalServerPort + private int port; + + @Autowired + private TestRestTemplate restTemplate; + + protected abstract String getTransformerName(); + + protected abstract String getSourceExtension(); + + @Test + public void testPageExists() + { + String result = restTemplate.getForObject("http://localhost:" + port + "/", String.class); + + String title = getTransformerName() + ' ' + "Test Transformation"; + assertTrue("\"" + title + "\" should be part of the page title", result.contains(title)); + } + + @Test + public void logPageExists() + { + String result = restTemplate.getForObject("http://localhost:" + port + "/log", + String.class); + + String title = getTransformerName() + ' ' + "Log"; + assertTrue("\"" + title + "\" should be part of the page title", result.contains(title)); + } + + @Test + public void errorPageExists() + { + String result = restTemplate.getForObject("http://localhost:" + port + "/error", + String.class); + + String title = getTransformerName() + ' ' + "Error Page"; + assertTrue("\"" + title + "\" should be part of the page title", + result.contains("Error Page")); + } + + @Test + public void noFileError() + { + LinkedMultiValueMap parameters = new LinkedMultiValueMap<>(); + parameters.add("targetExtension", ".tmp"); + + assertTransformError(false, + getTransformerName() + " - Required request part 'file' is not present", + parameters); + } + + @Test + public void noTargetExtensionError() + { + assertMissingParameter("targetExtension"); + } + + private void assertMissingParameter(String name) + { + assertTransformError(true, + getTransformerName() + " - Request parameter '" + name + "' is missing", null); + } + + protected void assertTransformError(boolean addFile, + String errorMessage, + LinkedMultiValueMap additionalParams) + { + LinkedMultiValueMap parameters = new LinkedMultiValueMap<>(); + if (addFile) + { + parameters.add("file", + new org.springframework.core.io.ClassPathResource("quick." + getSourceExtension())); + } + if (additionalParams != null) + { + parameters.addAll(additionalParams); + } + + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MULTIPART_FORM_DATA); + HttpEntity> entity = new HttpEntity<>(parameters, + headers); + + sendTranformationRequest(entity, errorMessage); + } + + @Test + public void httpTransformRequestDirectAccessUrlNotFoundTest() + { + String directUrl = "https://expired/direct/access/url"; + + LinkedMultiValueMap parameters = new LinkedMultiValueMap<>(); + parameters.add("targetExtension", ".tmp"); + parameters.add(DIRECT_ACCESS_URL, directUrl); + + assertTransformError(false, + getTransformerName() + " - Direct Access Url not found.", + parameters); + + } + + protected void sendTranformationRequest( + final HttpEntity> entity, final String errorMessage) + { + final ResponseEntity response = restTemplate.exchange(ENDPOINT_TRANSFORM, POST, entity, + String.class, ""); + assertEquals(errorMessage, getErrorMessage(response.getBody())); + } + + // Strip out just the error message from the returned json content body + // Had been expecting the Error page to be returned, but we end up with the json in this test harness. + // Is correct if run manually, so not worrying too much about this. + private String getErrorMessage(String content) + { + String message = ""; + int i = content.indexOf("\"message\":\""); + if (i != -1) + { + int j = content.indexOf("\",\"path\":", i); + if (j != -1) + { + message = content.substring(i + 11, j); + } + } + return message; + } +} diff --git a/t-engine-base/src/test/java/org/alfresco/transform/base/AbstractMetadataExtractsIT.java b/t-engine-base/src/test/java/org/alfresco/transform/base/AbstractMetadataExtractsIT.java new file mode 100644 index 00000000..b7903832 --- /dev/null +++ b/t-engine-base/src/test/java/org/alfresco/transform/base/AbstractMetadataExtractsIT.java @@ -0,0 +1,134 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.base; + +import static java.text.MessageFormat.format; +import static org.alfresco.transform.base.EngineClient.sendTRequest; +import static org.alfresco.transform.common.Mimetype.MIMETYPE_METADATA_EXTRACT; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; +import static org.springframework.http.HttpStatus.OK; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + +import org.springframework.core.io.Resource; +import org.springframework.http.ResponseEntity; + +/** + * Super class of metadata integration tests. Sub classes should provide the following: + *

+ *

    + *
  • A method providing a + * Stream of test files: {@code public static Stream engineTransformations()};
    • + *
  • Provide expected json files (<sourceFilename>"_metadata.json") as resources on the classpath.
    • + *
  • Override the method {@code testTransformation(TestFileInfo testFileInfo)} such that it calls + * the super method as a {@code @ParameterizedTest} for example:
+ * 
+ * @ParameterizedTest
+ * 
+ * @MethodSource("engineTransformations")
+ * 
+ * @Override
+
+ * public void testTransformation(TestFileInfo testFileInfo)
+ * 
+ * { 
+ *      super.testTransformation(TestFileInfo testFileInfo)
+ * }
+ *
+ * + * @author adavis + * @author dedwards + */ +public abstract class AbstractMetadataExtractsIT +{ + private static final String ENGINE_URL = "http://localhost:8090"; + // These are normally variable, hence the lowercase. + private static final String targetMimetype = MIMETYPE_METADATA_EXTRACT; + private static final String targetExtension = "json"; + + private final ObjectMapper jsonObjectMapper = new ObjectMapper(); + + + public void testTransformation(TestFileInfo testFileInfo) + { + final String sourceMimetype = testFileInfo.getMimeType(); + final String sourceFile = testFileInfo.getPath(); + + final String descriptor = format("Transform ({0}, {1} -> {2}, {3})", + sourceFile, sourceMimetype, targetMimetype, targetExtension); + + try + { + final ResponseEntity response = sendTRequest(ENGINE_URL, sourceFile, + sourceMimetype, targetMimetype, targetExtension); + assertEquals(OK, response.getStatusCode(), descriptor); + + String metadataFilename = sourceFile + "_metadata.json"; + Map actualMetadata = readMetadata(response.getBody().getInputStream()); + File actualMetadataFile = new File(metadataFilename); + jsonObjectMapper.writerWithDefaultPrettyPrinter().writeValue(actualMetadataFile, actualMetadata); + + Map expectedMetadata = readExpectedMetadata(metadataFilename, actualMetadataFile); + assertEquals(expectedMetadata, actualMetadata, + sourceFile+": The metadata did not match the expected value. It has been saved in "+actualMetadataFile.getAbsolutePath()); + actualMetadataFile.delete(); + } + catch (Exception e) + { + e.printStackTrace(); + fail(descriptor + " exception: " + e.getMessage()); + } + } + + private Map readExpectedMetadata(String filename, File actualMetadataFile) throws IOException + { + try (InputStream inputStream = this.getClass().getClassLoader().getResourceAsStream(filename)) + { + if (inputStream == null) + { + fail("The expected metadata file "+filename+" did not exist.\n"+ + "The actual metadata has been saved in "+actualMetadataFile.getAbsoluteFile()); + } + return readMetadata(inputStream); + } + } + + private Map readMetadata(InputStream inputStream) throws IOException + { + TypeReference> typeRef = new TypeReference>() {}; + return jsonObjectMapper.readValue(inputStream, typeRef); + } +} diff --git a/t-engine-base/src/test/java/org/alfresco/transform/base/AbstractQueueTransformServiceIT.java b/t-engine-base/src/test/java/org/alfresco/transform/base/AbstractQueueTransformServiceIT.java new file mode 100644 index 00000000..bb6d8d56 --- /dev/null +++ b/t-engine-base/src/test/java/org/alfresco/transform/base/AbstractQueueTransformServiceIT.java @@ -0,0 +1,74 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2021 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.base; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import javax.jms.Queue; + +import org.alfresco.transform.client.model.TransformReply; +import org.alfresco.transform.client.model.TransformRequest; +import org.apache.activemq.command.ActiveMQQueue; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.jms.core.JmsTemplate; + +/** + * @author Lucian Tuca + * created on 15/01/2019 + */ +@SpringBootTest(properties = {"activemq.url=nio://localhost:61616"}) +public abstract class AbstractQueueTransformServiceIT +{ + @Autowired + private Queue engineRequestQueue; + + @Autowired + private JmsTemplate jmsTemplate; + + private final ActiveMQQueue testingQueue = new ActiveMQQueue( + "org.alfresco.transform.engine.IT"); + + @Test + public void queueTransformServiceIT() + { + TransformRequest request = buildRequest(); + + jmsTemplate.convertAndSend(engineRequestQueue, request, m -> { + m.setJMSCorrelationID(request.getRequestId()); + m.setJMSReplyTo(testingQueue); + return m; + }); + + this.jmsTemplate.setReceiveTimeout(1_000); + TransformReply reply = (TransformReply) this.jmsTemplate.receiveAndConvert(testingQueue); + assertEquals(request.getRequestId(), reply.getRequestId()); + } + + protected abstract TransformRequest buildRequest(); +} diff --git a/t-engine-base/src/test/java/org/alfresco/transform/base/AbstractTransformerControllerTest.java b/t-engine-base/src/test/java/org/alfresco/transform/base/AbstractTransformerControllerTest.java new file mode 100644 index 00000000..d2dc1fc9 --- /dev/null +++ b/t-engine-base/src/test/java/org/alfresco/transform/base/AbstractTransformerControllerTest.java @@ -0,0 +1,686 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2022 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.base; + +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; +import static org.alfresco.transform.common.RequestParamMap.DIRECT_ACCESS_URL; +import static org.alfresco.transform.common.RequestParamMap.ENDPOINT_TRANSFORM; +import static org.alfresco.transform.common.RequestParamMap.ENDPOINT_TRANSFORM_CONFIG_LATEST; +import static org.alfresco.transform.common.RequestParamMap.ENDPOINT_TRANSFORM_CONFIG; +import static org.hamcrest.Matchers.containsString; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; +import static org.springframework.http.HttpHeaders.ACCEPT; +import static org.springframework.http.HttpHeaders.CONTENT_TYPE; +import static org.springframework.http.HttpStatus.BAD_REQUEST; +import static org.springframework.http.HttpStatus.CREATED; +import static org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR; +import static org.springframework.http.HttpStatus.OK; +import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.net.URL; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +import org.alfresco.transform.base.TransformController; +import org.alfresco.transform.base.probes.ProbeTestTransform; +import org.alfresco.transform.client.model.InternalContext; +import org.alfresco.transform.client.model.TransformReply; +import org.alfresco.transform.client.model.TransformRequest; +import org.alfresco.transform.config.SupportedSourceAndTarget; +import org.alfresco.transform.config.TransformConfig; +import org.alfresco.transform.config.TransformOption; +import org.alfresco.transform.config.TransformOptionGroup; +import org.alfresco.transform.config.TransformOptionValue; +import org.alfresco.transform.config.Transformer; +import org.alfresco.transform.registry.TransformServiceRegistry; +import org.alfresco.transform.messages.TransformStack; +import org.alfresco.transform.base.clients.AlfrescoSharedFileStoreClient; +import org.alfresco.transform.base.model.FileRefEntity; +import org.alfresco.transform.base.model.FileRefResponse; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.springframework.core.io.ClassPathResource; +import org.springframework.mock.web.MockMultipartFile; +import org.springframework.test.util.ReflectionTestUtils; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.ResultActions; +import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +/** + * Super class for testing controllers without a server. Includes tests for the Controller itself. + */ +public abstract class AbstractTransformerControllerTest +{ + @TempDir // added as part of ATS-702 to allow test resources to be read from the imported jar files to prevent test resource duplication + public File tempDir; + + @Autowired + TransformController controller; + + @Autowired + protected MockMvc mockMvc; + + @Autowired + protected ObjectMapper objectMapper; + + @MockBean + protected AlfrescoSharedFileStoreClient alfrescoSharedFileStoreClient; + + @SpyBean + protected TransformServiceRegistry transformRegistry; + + @Value("${transform.core.version}") + private String coreVersion; + + protected String sourceExtension; + protected String targetExtension; + protected String sourceMimetype; + protected String targetMimetype; + protected HashMap options = new HashMap<>(); + + protected MockMultipartFile sourceFile; + protected String expectedOptions; + protected String expectedSourceSuffix; + protected Long expectedTimeout = 0L; + protected byte[] expectedSourceFileBytes; + + /** + * The expected result. Taken resting target quick file's bytes. + * + * Note: These checks generally don't work on Windows (Mac and Linux are okay). Possibly to do with byte order + * loading. + */ + protected byte[] expectedTargetFileBytes; + + // Called by sub class + protected abstract void mockTransformCommand(String sourceExtension, + String targetExtension, String sourceMimetype, + boolean readTargetFileBytes) throws IOException; + + protected TransformController getController() + { + return controller; + } + + protected ProbeTestTransform getProbeTestTransform() + { + return controller.probeTestTransform; + } + + protected abstract void updateTransformRequestWithSpecificOptions( + TransformRequest transformRequest); + + /** + * This method ends up being the core of the mock. + * It copies content from an existing file in the resources folder to the desired location + * in order to simulate a successful transformation. + * + * @param actualTargetExtension Requested extension. + * @param testFile The test file (transformed) - basically the result. + * @param targetFile The location where the content from the testFile should be copied + * @throws IOException in case of any errors. + */ + public void generateTargetFileFromResourceFile(String actualTargetExtension, File testFile, + File targetFile) throws IOException + { + if (testFile != null) + { + try (var inputStream = new FileInputStream(testFile); + var outputStream = new FileOutputStream(targetFile)) + { + FileChannel source = inputStream.getChannel(); + FileChannel target = outputStream.getChannel(); + target.transferFrom(source, 0, source.size()); + + } catch (Exception e) + { + throw e; + } + } + else + { + testFile = getTestFile("quick." + actualTargetExtension, false); + if (testFile != null) + { + try (var inputStream = new FileInputStream(testFile); + var outputStream = new FileOutputStream(targetFile)) + { + FileChannel source = inputStream.getChannel(); + FileChannel target = outputStream.getChannel(); + target.transferFrom(source, 0, source.size()); + + } catch (Exception e) + { + throw e; + } + } + } + } + + protected byte[] readTestFile(String extension) throws IOException + { + return Files.readAllBytes(getTestFile("quick." + extension, true).toPath()); + } + + protected File getTestFile(String testFilename, boolean required) throws IOException + { + File testFile = null; + ClassLoader classLoader = getClass().getClassLoader(); + URL testFileUrl = classLoader.getResource(testFilename); + if (required && testFileUrl == null) + { + throw new IOException("The test file " + testFilename + + " does not exist in the resources directory"); + } + // added as part of ATS-702 to allow test resources to be read from the imported jar files to prevent test resource duplication + if (testFileUrl!=null) + { + // Each use of the tempDir should result in a unique directory being used + testFile = new File(tempDir, testFilename); + Files.copy(classLoader.getResourceAsStream(testFilename), testFile.toPath(),REPLACE_EXISTING); + } + + return testFileUrl == null ? null : testFile; + } + + protected MockHttpServletRequestBuilder mockMvcRequest(String url, MockMultipartFile sourceFile, + String... params) + { + if (sourceFile == null) + { + return mockMvcRequestWithoutMockMultipartFile(url, params); + } + else + { + return mockMvcRequestWithMockMultipartFile(url, sourceFile, params); + } + } + + private MockHttpServletRequestBuilder mockMvcRequestWithoutMockMultipartFile(String url, + String... params) + { + MockHttpServletRequestBuilder builder = MockMvcRequestBuilders.multipart(ENDPOINT_TRANSFORM); + + if (params.length % 2 != 0) + { + throw new IllegalArgumentException("each param should have a name and value."); + } + for (int i = 0; i < params.length; i += 2) + { + builder = builder.param(params[i], params[i + 1]); + } + + return builder; + } + + private MockHttpServletRequestBuilder mockMvcRequestWithMockMultipartFile(String url, MockMultipartFile sourceFile, + String... params) + { + MockHttpServletRequestBuilder builder = MockMvcRequestBuilders.multipart(ENDPOINT_TRANSFORM).file( + sourceFile); + + if (params.length % 2 != 0) + { + throw new IllegalArgumentException("each param should have a name and value."); + } + for (int i = 0; i < params.length; i += 2) + { + builder = builder.param(params[i], params[i + 1]); + } + + return builder; + } + + protected TransformRequest createTransformRequest(String sourceFileRef, File sourceFile) + { + TransformRequest transformRequest = new TransformRequest(); + transformRequest.setRequestId("1"); + transformRequest.setSchema(1); + transformRequest.setClientData("Alfresco Digital Business Platform"); + transformRequest.setTransformRequestOptions(options); + transformRequest.setSourceReference(sourceFileRef); + transformRequest.setSourceExtension(sourceExtension); + transformRequest.setSourceMediaType(sourceMimetype); + transformRequest.setSourceSize(sourceFile.length()); + transformRequest.setTargetExtension(targetExtension); + transformRequest.setTargetMediaType(targetMimetype); + transformRequest.setInternalContext(InternalContext.initialise(null)); + transformRequest.getInternalContext().getMultiStep().setInitialRequestId("123"); + transformRequest.getInternalContext().getMultiStep().setInitialSourceMediaType(sourceMimetype); + TransformStack.setInitialTransformRequestOptions(transformRequest.getInternalContext(), options); + TransformStack.setInitialSourceReference(transformRequest.getInternalContext(), sourceFileRef); + TransformStack.addTransformLevel(transformRequest.getInternalContext(), + TransformStack.levelBuilder(TransformStack.PIPELINE_FLAG) + .withStep("transformerName", sourceMimetype, targetMimetype)); + return transformRequest; + } + + @Test + public void simpleTransformTest() throws Exception + { + mockMvc.perform( + mockMvcRequest(ENDPOINT_TRANSFORM, sourceFile, "targetExtension", targetExtension)) + .andExpect(status().is(OK.value())) + .andExpect(content().bytes(expectedTargetFileBytes)) + .andExpect(header().string("Content-Disposition", + "attachment; filename*= UTF-8''quick." + targetExtension)); + } + + @Test + public void testDelayTest() throws Exception + { + long start = System.currentTimeMillis(); + mockMvc.perform(mockMvcRequest(ENDPOINT_TRANSFORM, sourceFile, "targetExtension", targetExtension, + "testDelay", "400")) + .andExpect(status().is(OK.value())) + .andExpect(content().bytes(expectedTargetFileBytes)) + .andExpect(header().string("Content-Disposition", + "attachment; filename*= UTF-8''quick." + targetExtension)); + long ms = System.currentTimeMillis() - start; + System.out.println("Transform incluing test delay was " + ms); + assertTrue(ms >= 400, "Delay sending the result back was too small " + ms); + assertTrue(ms <= 500,"Delay sending the result back was too big " + ms); + } + + @Test + public void noTargetFileTest() throws Exception + { + mockMvc.perform(mockMvcRequest(ENDPOINT_TRANSFORM, sourceFile, "targetExtension", "xxx")) + .andExpect(status().is(INTERNAL_SERVER_ERROR.value())); + } + + @Test + // Looks dangerous but is okay as we only use the final filename + public void dotDotSourceFilenameTest() throws Exception + { + sourceFile = new MockMultipartFile("file", "../quick." + sourceExtension, sourceMimetype, + expectedSourceFileBytes); + + mockMvc.perform( + mockMvcRequest(ENDPOINT_TRANSFORM, sourceFile, "targetExtension", targetExtension)) + .andExpect(status().is(OK.value())) + .andExpect(content().bytes(expectedTargetFileBytes)) + .andExpect(header().string("Content-Disposition", + "attachment; filename*= UTF-8''quick." + targetExtension)); + } + + @Test + // Is okay, as the target filename is built up from the whole source filename and the targetExtension + public void noExtensionSourceFilenameTest() throws Exception + { + sourceFile = new MockMultipartFile("file", "../quick", sourceMimetype, + expectedSourceFileBytes); + + mockMvc.perform( + mockMvcRequest(ENDPOINT_TRANSFORM, sourceFile, "targetExtension", targetExtension)) + .andExpect(status().is(OK.value())) + .andExpect(content().bytes(expectedTargetFileBytes)) + .andExpect(header().string("Content-Disposition", + "attachment; filename*= UTF-8''quick." + targetExtension)); + } + + @Test + // Invalid file name that ends in / + public void badSourceFilenameTest() throws Exception + { + sourceFile = new MockMultipartFile("file", "abc/", sourceMimetype, expectedSourceFileBytes); + + mockMvc.perform( + mockMvcRequest(ENDPOINT_TRANSFORM, sourceFile, "targetExtension", targetExtension)) + .andExpect(status().is(BAD_REQUEST.value())) + .andExpect(status().reason(containsString("The source filename was not supplied"))); + } + + @Test + public void blankSourceFilenameTest() throws Exception + { + sourceFile = new MockMultipartFile("file", "", sourceMimetype, expectedSourceFileBytes); + + mockMvc.perform( + mockMvcRequest(ENDPOINT_TRANSFORM, sourceFile, "targetExtension", targetExtension)) + .andExpect(status().is(BAD_REQUEST.value())); + } + + @Test + public void noTargetExtensionTest() throws Exception + { + mockMvc.perform(mockMvcRequest(ENDPOINT_TRANSFORM, sourceFile)) + .andExpect(status().is(BAD_REQUEST.value())) + .andExpect(status().reason( + containsString("Request parameter 'targetExtension' is missing"))); + } + + @Test + public void calculateMaxTime() throws Exception + { + ProbeTestTransform probeTestTransform = getController().probeTestTransform; + probeTestTransform.setLivenessPercent(110); + + long[][] values = new long[][]{ + {5000, 0, Long.MAX_VALUE}, // 1st transform is ignored + {1000, 1000, 2100}, // 1000 + 1000*1.1 + {3000, 2000, 4200}, // 2000 + 2000*1.1 + {2000, 2000, 4200}, + {6000, 3000, 6300}, + {8000, 4000, 8400}, + {4444, 4000, 8400}, // no longer in the first few, so normal and max times don't change + {5555, 4000, 8400} + }; + + for (long[] v : values) + { + long time = v[0]; + long expectedNormalTime = v[1]; + long expectedMaxTime = v[2]; + + probeTestTransform.calculateMaxTime(time, true); + assertEquals(expectedNormalTime, probeTestTransform.getNormalTime()); + assertEquals(expectedMaxTime, probeTestTransform.getMaxTime()); + } + } + + @Test + public void testEmptyPojoTransform() throws Exception + { + // Transformation Request POJO + TransformRequest transformRequest = new TransformRequest(); + + // Serialize and call the transformer + String tr = objectMapper.writeValueAsString(transformRequest); + String transformationReplyAsString = mockMvc + .perform(MockMvcRequestBuilders + .post(ENDPOINT_TRANSFORM) + .header(ACCEPT, APPLICATION_JSON_VALUE) + .header(CONTENT_TYPE, APPLICATION_JSON_VALUE) + .content(tr)) + .andExpect(status().is(BAD_REQUEST.value())) + .andReturn().getResponse().getContentAsString(); + + TransformReply transformReply = objectMapper.readValue(transformationReplyAsString, + TransformReply.class); + + // Assert the reply + assertEquals(BAD_REQUEST.value(), transformReply.getStatus()); + } + + /** + * + * @return transformer specific engine config name + */ + public String getEngineConfigName() + { + return "engine_config.json"; + } + + @Test + public void testGetTransformConfigInfo() throws Exception + { + TransformConfig expectedTransformConfig = objectMapper + .readValue(getTestFile(getEngineConfigName(), true), + TransformConfig.class); + expectedTransformConfig.getTransformers().forEach(transformer -> { + transformer.setCoreVersion(coreVersion); + transformer.getTransformOptions().add(DIRECT_ACCESS_URL); + }); + expectedTransformConfig.getTransformOptions().put(DIRECT_ACCESS_URL, Set.of(new TransformOptionValue(false, DIRECT_ACCESS_URL))); + + ReflectionTestUtils.setField(transformRegistry, "engineConfig", + new ClassPathResource(getEngineConfigName())); + + String response = mockMvc + .perform(MockMvcRequestBuilders.get(ENDPOINT_TRANSFORM_CONFIG_LATEST)) + .andExpect(status().is(OK.value())) + .andExpect(header().string(CONTENT_TYPE, APPLICATION_JSON_VALUE)) + .andReturn().getResponse().getContentAsString(); + + TransformConfig transformConfig = objectMapper.readValue(response, TransformConfig.class); + assertEquals(expectedTransformConfig, transformConfig); + } + + @Test + // Test for case when T-Router or Repository is a version that does not expect it + public void testGetTransformConfigInfoExcludingCoreVersion() throws Exception + { + TransformConfig expectedTransformConfig = objectMapper + .readValue(getTestFile(getEngineConfigName(), true), + TransformConfig.class); + + ReflectionTestUtils.setField(transformRegistry, "engineConfig", + new ClassPathResource(getEngineConfigName())); + + String response = mockMvc + .perform(MockMvcRequestBuilders.get(ENDPOINT_TRANSFORM_CONFIG)) + .andExpect(status().is(OK.value())) + .andExpect(header().string(CONTENT_TYPE, APPLICATION_JSON_VALUE)) + .andReturn().getResponse().getContentAsString(); + + TransformConfig transformConfig = objectMapper.readValue(response, TransformConfig.class); + assertEquals(expectedTransformConfig, transformConfig); + } + + @Test + public void testGetInfoFromConfigWithDuplicates() throws Exception + { + TransformConfig expectedResult = buildCompleteTransformConfig(); + + ReflectionTestUtils.setField(transformRegistry, "engineConfig", + new ClassPathResource("engine_config_with_duplicates.json")); + + String response = mockMvc + .perform(MockMvcRequestBuilders.get(ENDPOINT_TRANSFORM_CONFIG)) + .andExpect(status().is(OK.value())) + .andExpect(header().string(CONTENT_TYPE, APPLICATION_JSON_VALUE)) + .andReturn().getResponse().getContentAsString(); + + TransformConfig transformConfig = objectMapper.readValue(response, TransformConfig.class); + + assertNotNull(transformConfig); + assertEquals(expectedResult, transformConfig); + assertEquals(3, transformConfig.getTransformOptions().get("engineXOptions").size()); + assertEquals(1, + transformConfig.getTransformers().get(0).getSupportedSourceAndTargetList().size()); + assertEquals(1, + transformConfig.getTransformers().get(0).getTransformOptions().size()); + } + + @Test + public void testGetInfoFromConfigWithEmptyTransformOptions() throws Exception + { + Transformer transformer = buildTransformer("application/pdf", "image/png"); + TransformConfig expectedResult = new TransformConfig(); + expectedResult.setTransformers(ImmutableList.of(transformer)); + + ReflectionTestUtils.setField(transformRegistry, "engineConfig", + new ClassPathResource("engine_config_incomplete.json")); + + String response = mockMvc + .perform(MockMvcRequestBuilders.get(ENDPOINT_TRANSFORM_CONFIG)) + .andExpect(status().is(OK.value())) + .andExpect(header().string(CONTENT_TYPE, APPLICATION_JSON_VALUE)) + .andReturn().getResponse().getContentAsString(); + + TransformConfig transformConfig = objectMapper.readValue(response, TransformConfig.class); + + assertNotNull(transformConfig); + assertEquals(expectedResult, transformConfig); + } + + @Test + public void testGetInfoFromConfigWithNoTransformOptions() throws Exception + { + Transformer transformer = buildTransformer("application/pdf", "image/png"); + transformer.setTransformerName("engineX"); + TransformConfig expectedResult = new TransformConfig(); + expectedResult.setTransformers(ImmutableList.of(transformer)); + + ReflectionTestUtils.setField(transformRegistry, "engineConfig", + new ClassPathResource("engine_config_no_transform_options.json")); + + String response = mockMvc + .perform(MockMvcRequestBuilders.get(ENDPOINT_TRANSFORM_CONFIG)) + .andExpect(status().is(OK.value())) + .andExpect(header().string(CONTENT_TYPE, APPLICATION_JSON_VALUE)) + .andReturn().getResponse().getContentAsString(); + + TransformConfig transformConfig = objectMapper.readValue(response, TransformConfig.class); + + assertNotNull(transformConfig); + assertEquals(expectedResult, transformConfig); + } + + private TransformConfig buildCompleteTransformConfig() + { + TransformConfig expectedResult = new TransformConfig(); + + Set transformOptionGroup = ImmutableSet.of( + new TransformOptionValue(false, "cropGravity")); + Set transformOptions = ImmutableSet.of( + new TransformOptionValue(false, "page"), + new TransformOptionValue(false, "width"), + new TransformOptionGroup(false, transformOptionGroup)); + Map> transformOptionsMap = ImmutableMap.of("engineXOptions", + transformOptions); + + Transformer transformer = buildTransformer("application/pdf", "image/png", "engineXOptions", + "engineX"); + List transformers = ImmutableList.of(transformer); + + expectedResult.setTransformOptions(transformOptionsMap); + expectedResult.setTransformers(transformers); + + return expectedResult; + } + + private Transformer buildTransformer(String sourceMediaType, String targetMediaType, + String transformOptions, String transformerName) + { + Transformer transformer = buildTransformer(sourceMediaType, targetMediaType); + transformer.setTransformerName(transformerName); + transformer.setTransformOptions(ImmutableSet.of(transformOptions)); + + return transformer; + } + + private Transformer buildTransformer(String sourceMediaType, String targetMediaType) + { + Set supportedSourceAndTargetList = ImmutableSet.of( + SupportedSourceAndTarget.builder() + .withSourceMediaType(sourceMediaType) + .withTargetMediaType(targetMediaType) + .build()); + + Transformer transformer = new Transformer(); + transformer.setSupportedSourceAndTargetList(supportedSourceAndTargetList); + return transformer; + } + + @Test + public void queueTransformRequestUsingDirectAccessUrlTest() throws Exception + { + // Files + String sourceFileRef = UUID.randomUUID().toString(); + File sourceFile = getTestFile("quick." + sourceExtension, true); + String targetFileRef = UUID.randomUUID().toString(); + + TransformRequest transformRequest = createTransformRequest(sourceFileRef, sourceFile); + Map transformRequestOptions = transformRequest.getTransformRequestOptions(); + + String directUrl = "file://" + sourceFile.toPath(); + + transformRequestOptions.put(DIRECT_ACCESS_URL, directUrl); + transformRequest.setTransformRequestOptions(transformRequestOptions); + + when(alfrescoSharedFileStoreClient.saveFile(any())) + .thenReturn(new FileRefResponse(new FileRefEntity(targetFileRef))); + + // Update the Transformation Request with any specific params before sending it + updateTransformRequestWithSpecificOptions(transformRequest); + + // Serialize and call the transformer + String tr = objectMapper.writeValueAsString(transformRequest); + String transformationReplyAsString = mockMvc + .perform(MockMvcRequestBuilders + .post("/transform") + .header(ACCEPT, APPLICATION_JSON_VALUE) + .header(CONTENT_TYPE, APPLICATION_JSON_VALUE) + .content(tr)) + .andExpect(status().is(CREATED.value())) + .andReturn().getResponse().getContentAsString(); + + TransformReply transformReply = objectMapper.readValue(transformationReplyAsString, + TransformReply.class); + + // Assert the reply + assertEquals(transformRequest.getRequestId(), transformReply.getRequestId()); + assertEquals(transformRequest.getClientData(), transformReply.getClientData()); + assertEquals(transformRequest.getSchema(), transformReply.getSchema()); + } + + @Test + public void httpTransformRequestUsingDirectAccessUrlTest() throws Exception + { + File dauSourceFile = getTestFile("quick." + sourceExtension, true); + String directUrl = "file://" + dauSourceFile.toPath(); + + ResultActions resultActions = mockMvc.perform( + mockMvcRequest(ENDPOINT_TRANSFORM, null) + .param("targetExtension", targetExtension) + .param(DIRECT_ACCESS_URL, directUrl)) + .andExpect(status().is(OK.value())); + + if (expectedTargetFileBytes != null) + { + resultActions.andExpect(content().bytes(expectedTargetFileBytes)); + } + } +} diff --git a/t-engine-base/src/test/java/org/alfresco/transform/base/EngineClient.java b/t-engine-base/src/test/java/org/alfresco/transform/base/EngineClient.java new file mode 100644 index 00000000..26c827f0 --- /dev/null +++ b/t-engine-base/src/test/java/org/alfresco/transform/base/EngineClient.java @@ -0,0 +1,69 @@ +/* + * Copyright 2015-2022 Alfresco Software, Ltd. All rights reserved. + * + * License rights for this program may be obtained from Alfresco Software, Ltd. + * pursuant to a written agreement and any use of this program without such an + * agreement is prohibited. + */ +package org.alfresco.transform.base; + +import static java.util.Collections.emptyMap; +import static org.alfresco.transform.common.RequestParamMap.ENDPOINT_TRANSFORM; +import static org.springframework.http.MediaType.MULTIPART_FORM_DATA; + +import java.util.Map; + +import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.Resource; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.ResponseEntity; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.client.RestTemplate; + +/** + * @author Cezar Leahu + */ +public class EngineClient +{ + private static final RestTemplate REST_TEMPLATE = new RestTemplate(); + + public static ResponseEntity sendTRequest( + final String engineUrl, final String sourceFile, + final String sourceMimetype, final String targetMimetype, final String targetExtension) + { + return sendTRequest(engineUrl, sourceFile, sourceMimetype, targetMimetype, targetExtension, + emptyMap()); + } + + public static ResponseEntity sendTRequest( + final String engineUrl, final String sourceFile, + final String sourceMimetype, final String targetMimetype, final String targetExtension, + final Map transformOptions) + { + final HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MULTIPART_FORM_DATA); + //headers.setAccept(ImmutableList.of(MULTIPART_FORM_DATA)); + + final MultiValueMap body = new LinkedMultiValueMap<>(); + body.add("file", new ClassPathResource(sourceFile)); + if (sourceMimetype != null && !sourceMimetype.trim().isEmpty()) + { + body.add("sourceMimetype", sourceMimetype); + } + if (targetMimetype != null && !targetMimetype.trim().isEmpty()) + { + body.add("targetMimetype", targetMimetype); + } + if (targetExtension != null && !targetExtension.trim().isEmpty()) + { + body.add("targetExtension", targetExtension); + } + transformOptions.forEach(body::add); + + final HttpEntity> entity = new HttpEntity<>(body, headers); + + return REST_TEMPLATE.postForEntity(engineUrl + ENDPOINT_TRANSFORM, entity, Resource.class); + } +} diff --git a/t-engine-base/src/test/java/org/alfresco/transform/base/QueueTransformServiceTest.java b/t-engine-base/src/test/java/org/alfresco/transform/base/QueueTransformServiceTest.java new file mode 100644 index 00000000..3e28ec9c --- /dev/null +++ b/t-engine-base/src/test/java/org/alfresco/transform/base/QueueTransformServiceTest.java @@ -0,0 +1,240 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2021 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ + +package org.alfresco.transform.base; + +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.springframework.http.HttpStatus.BAD_REQUEST; +import static org.springframework.http.HttpStatus.CREATED; +import static org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR; + +import javax.jms.Destination; +import javax.jms.JMSException; +import javax.jms.Message; + +import org.alfresco.transform.client.model.TransformReply; +import org.alfresco.transform.client.model.TransformRequest; +import org.alfresco.transform.base.messaging.TransformMessageConverter; +import org.alfresco.transform.base.messaging.TransformReplySender; +import org.apache.activemq.command.ActiveMQObjectMessage; +import org.apache.activemq.command.ActiveMQQueue; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.jms.support.converter.MessageConversionException; + +public class QueueTransformServiceTest +{ + @Mock + private TransformController transformController; + @Mock + private TransformMessageConverter transformMessageConverter; + @Mock + private TransformReplySender transformReplySender; + + @InjectMocks + private QueueTransformService queueTransformService; + + @BeforeEach + public void setup() + { + MockitoAnnotations.initMocks(this); + } + + @Test + public void testWhenReceiveNullMessageThenStopFlow() + { + queueTransformService.receive(null); + + verifyNoMoreInteractions(transformController); + verifyNoMoreInteractions(transformMessageConverter); + verifyNoMoreInteractions(transformReplySender); + } + + @Test + public void testWhenReceiveMessageWithNoReplyToQueueThenStopFlow() + { + queueTransformService.receive(new ActiveMQObjectMessage()); + + verifyNoMoreInteractions(transformController); + verifyNoMoreInteractions(transformMessageConverter); + verifyNoMoreInteractions(transformReplySender); + } + + @Test + public void testConvertMessageReturnsNullThenReplyWithInternalServerError() throws JMSException + { + ActiveMQObjectMessage msg = new ActiveMQObjectMessage(); + msg.setCorrelationId("1234"); + ActiveMQQueue destination = new ActiveMQQueue(); + msg.setJMSReplyTo(destination); + + TransformReply reply = TransformReply + .builder() + .withStatus(INTERNAL_SERVER_ERROR.value()) + .withErrorDetails( + "JMS exception during T-Request deserialization of message with correlationID " + + msg.getCorrelationId() + ": null") + .build(); + + doReturn(null).when(transformMessageConverter).fromMessage(msg); + + queueTransformService.receive(msg); + + verify(transformMessageConverter).fromMessage(msg); + verify(transformReplySender).send(destination, reply, msg.getCorrelationId()); + + verifyNoMoreInteractions(transformController); + } + + @Test + public void testConvertMessageThrowsMessageConversionExceptionThenReplyWithBadRequest() + throws JMSException + { + ActiveMQObjectMessage msg = new ActiveMQObjectMessage(); + msg.setCorrelationId("1234"); + ActiveMQQueue destination = new ActiveMQQueue(); + msg.setJMSReplyTo(destination); + + TransformReply reply = TransformReply + .builder() + .withStatus(BAD_REQUEST.value()) + .withErrorDetails( + "Message conversion exception during T-Request deserialization of message with correlationID" + + msg.getCorrelationId() + ": null") + .build(); + + doThrow(MessageConversionException.class).when(transformMessageConverter).fromMessage(msg); + + queueTransformService.receive(msg); + + verify(transformMessageConverter).fromMessage(msg); + verify(transformReplySender).send(destination, reply, msg.getCorrelationId()); + + verifyNoMoreInteractions(transformController); + } + + @Test + public void testConvertMessageThrowsJMSExceptionThenReplyWithInternalServerError() + throws JMSException + { + ActiveMQObjectMessage msg = new ActiveMQObjectMessage(); + msg.setCorrelationId("1234"); + ActiveMQQueue destination = new ActiveMQQueue(); + msg.setJMSReplyTo(destination); + + TransformReply reply = TransformReply + .builder() + .withStatus(INTERNAL_SERVER_ERROR.value()) + .withErrorDetails( + "JMSException during T-Request deserialization of message with correlationID " + + msg.getCorrelationId() + ": null") + .build(); + + doThrow(JMSException.class).when(transformMessageConverter).fromMessage(msg); + + queueTransformService.receive(msg); + + verify(transformMessageConverter).fromMessage(msg); + verify(transformReplySender).send(destination, reply, msg.getCorrelationId()); + + verifyNoMoreInteractions(transformController); + } + + @Test + public void testWhenReceiveValidTransformRequestThenReplyWithSuccess() throws JMSException + { + ActiveMQObjectMessage msg = new ActiveMQObjectMessage(); + ActiveMQQueue destination = new ActiveMQQueue(); + msg.setJMSReplyTo(destination); + + TransformRequest request = new TransformRequest(); + TransformReply reply = TransformReply + .builder() + .withStatus(CREATED.value()) + .build(); + + doReturn(request).when(transformMessageConverter).fromMessage(msg); + doReturn(new ResponseEntity<>(reply, HttpStatus.valueOf(reply.getStatus()))) + .when(transformController).transform(request, null); + + queueTransformService.receive(msg); + + verify(transformMessageConverter).fromMessage(msg); + verify(transformController).transform(request, null); + verify(transformReplySender).send(destination, reply); + } + + @Test + public void testWhenJMSExceptionOnMessageIsThrownThenStopFlow() throws JMSException + { + Message msg = mock(Message.class); + + doThrow(JMSException.class).when(msg).getJMSReplyTo(); + + queueTransformService.receive(msg); + + verifyNoMoreInteractions(transformController); + verifyNoMoreInteractions(transformMessageConverter); + verifyNoMoreInteractions(transformReplySender); + } + + @Test + public void testWhenExceptionOnCorrelationIdIsThrownThenContinueFlowWithNullCorrelationId() + throws JMSException + { + Message msg = mock(Message.class); + Destination destination = mock(Destination.class); + + doThrow(JMSException.class).when(msg).getJMSCorrelationID(); + doReturn(destination).when(msg).getJMSReplyTo(); + + TransformRequest request = new TransformRequest(); + TransformReply reply = TransformReply + .builder() + .withStatus(CREATED.value()) + .build(); + + doReturn(request).when(transformMessageConverter).fromMessage(msg); + doReturn(new ResponseEntity<>(reply, HttpStatus.valueOf(reply.getStatus()))) + .when(transformController).transform(request, null); + + queueTransformService.receive(msg); + + verify(transformMessageConverter).fromMessage(msg); + verify(transformController).transform(request, null); + verify(transformReplySender).send(destination, reply); + } +} diff --git a/t-engine-base/src/test/java/org/alfresco/transform/base/SourceTarget.java b/t-engine-base/src/test/java/org/alfresco/transform/base/SourceTarget.java new file mode 100644 index 00000000..8a55ef7f --- /dev/null +++ b/t-engine-base/src/test/java/org/alfresco/transform/base/SourceTarget.java @@ -0,0 +1,76 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2019 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.base; + +import java.util.Objects; + +/** + * @author Cezar Leahu + */ +public class SourceTarget +{ + final String source; + final String target; + + private SourceTarget(final String source, final String target) + { + this.source = source; + this.target = target; + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + SourceTarget that = (SourceTarget) o; + return Objects.equals(source, that.source) && + Objects.equals(target, that.target); + } + + @Override + public int hashCode() + { + return Objects.hash(source, target); + } + + @Override + public String toString() + { + return source + '|' + target; + } + + public static SourceTarget of(final String source, final String target) + { + return sourceTarget(source, target); + } + + public static SourceTarget sourceTarget(final String source, final String target) + { + return new SourceTarget(source, target); + } +} diff --git a/t-engine-base/src/test/java/org/alfresco/transform/base/TestFileInfo.java b/t-engine-base/src/test/java/org/alfresco/transform/base/TestFileInfo.java new file mode 100644 index 00000000..37100de8 --- /dev/null +++ b/t-engine-base/src/test/java/org/alfresco/transform/base/TestFileInfo.java @@ -0,0 +1,85 @@ +/* + * #%L + * Alfresco Transform Core + * %% + * Copyright (C) 2005 - 2019 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * - + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * - + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * - + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * - + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.transform.base; + +/** + * @author Cezar Leahu + */ +public class TestFileInfo +{ + private final String mimeType; + private final String extension; + private final String path; + private final boolean exactMimeType; + + public TestFileInfo(final String mimeType, final String extension, final String path, + final boolean exactMimeType) + { + this.mimeType = mimeType; + this.extension = extension; + this.path = path; + this.exactMimeType = exactMimeType; + } + + public String getMimeType() + { + return mimeType; + } + + public String getExtension() + { + return extension; + } + + public String getPath() + { + return path; + } + + public boolean isExactMimeType() + { + return exactMimeType; + } + + public static TestFileInfo testFile(final String mimeType, final String extension, + final String path, final boolean exactMimeType) + { + return new TestFileInfo(mimeType, extension, path, exactMimeType); + } + + public static TestFileInfo testFile(final String mimeType, final String extension, + final String path) + { + return new TestFileInfo(mimeType, extension, path, false); + } + + @Override + public String toString() + { + return path; + } +} diff --git a/t-engine-base/src/test/resources/engine_config_complete.json b/t-engine-base/src/test/resources/engine_config_complete.json new file mode 100644 index 00000000..0ab0ea15 --- /dev/null +++ b/t-engine-base/src/test/resources/engine_config_complete.json @@ -0,0 +1,22 @@ +{ + "transformOptions": { + "engineXOptions": [ + {"value": {"name": "page"}}, + {"value": {"name": "width"}}, + {"group": {"transformOptions": [ + {"value": {"name": "cropGravity"}} + ]}} + ] + }, + "transformers": [ + { + "transformerName": "engineX", + "supportedSourceAndTargetList": [ + {"sourceMediaType": "application/pdf", "targetMediaType": "image/png" } + ], + "transformOptions": [ + "engineXOptions" + ] + } + ] +} \ No newline at end of file diff --git a/t-engine-base/src/test/resources/engine_config_incomplete.json b/t-engine-base/src/test/resources/engine_config_incomplete.json new file mode 100644 index 00000000..04935a7b --- /dev/null +++ b/t-engine-base/src/test/resources/engine_config_incomplete.json @@ -0,0 +1,10 @@ +{ + "transformOptions": {}, + "transformers": [ + { + "supportedSourceAndTargetList": [ + {"sourceMediaType": "application/pdf", "targetMediaType": "image/png" } + ] + } + ] +} \ No newline at end of file diff --git a/t-engine-base/src/test/resources/engine_config_no_transform_options.json b/t-engine-base/src/test/resources/engine_config_no_transform_options.json new file mode 100644 index 00000000..1ceebe95 --- /dev/null +++ b/t-engine-base/src/test/resources/engine_config_no_transform_options.json @@ -0,0 +1,10 @@ +{ + "transformers": [ + { + "transformerName": "engineX", + "supportedSourceAndTargetList": [ + {"sourceMediaType": "application/pdf", "targetMediaType": "image/png" } + ] + } + ] +} \ No newline at end of file diff --git a/t-engine-base/src/test/resources/engine_config_with_duplicates.json b/t-engine-base/src/test/resources/engine_config_with_duplicates.json new file mode 100644 index 00000000..581ad9cc --- /dev/null +++ b/t-engine-base/src/test/resources/engine_config_with_duplicates.json @@ -0,0 +1,26 @@ +{ + "transformOptions": { + "engineXOptions": [ + {"value": {"name": "page"}}, + {"value": {"name": "page"}}, + {"value": {"name": "width"}}, + {"group": {"transformOptions": [ + {"value": {"name": "cropGravity"}} + ]}} + ] + }, + "transformers": [ + { + "transformerName": "engineX", + "supportedSourceAndTargetList": [ + {"sourceMediaType": "application/pdf", "targetMediaType": "image/png" }, + {"sourceMediaType": "application/pdf", "targetMediaType": "image/png" }, + {"sourceMediaType": "application/pdf", "targetMediaType": "image/png" } + ], + "transformOptions": [ + "engineXOptions", + "engineXOptions" + ] + } + ] +} \ No newline at end of file