inteligr8/alfresco-community-repo
1
0
Fork 0
mirror of https://github.com/Alfresco/alfresco-community-repo.git synced 2025-09-17 14:21:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

REPO-5208 Metadata extract: overwritePolicy, enableStringTagging and carryAspectProperties tests (#116)

Browse Source 
* REPO-5208 Addition of extra async metadata extract tests for overriding policy, tag extractio and carryAspectProperties

Main author: Adina Ababei <adina.ababei@ness.com>
Testing of tagging modified by Alan Davis to pass when Solr is not running.
This commit is contained in:
Alan Davis
2020-11-11 15:29:08 +00:00
committed by GitHub
parent cdf2630bde
commit cca20b3b18
11 changed files with 358 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

333
repository/src/test/java/org/alfresco/repo/content/metadata/AsynchronousExtractorTest.java → repository/src/test/java/org/alfresco/repo/action/executer/AsynchronousExtractorTest.java
View File

@@ -30,6 +30,7 @@ import org.alfresco.repo.action.ActionImpl;
import org.alfresco.repo.content.MimetypeMap;
import org.alfresco.repo.content.metadata.AbstractMappingMetadataExtracter;
import org.alfresco.repo.content.metadata.AsynchronousExtractor;
import org.alfresco.repo.content.metadata.MetadataExtracter.OverwritePolicy;
import org.alfresco.repo.content.metadata.MetadataExtracterRegistry;
import org.alfresco.repo.content.transform.AbstractContentTransformerTest;
import org.alfresco.repo.content.transform.TransformerDebug;
@@ -37,7 +38,9 @@ import org.alfresco.repo.content.transform.UnsupportedTransformationException;
import org.alfresco.repo.rendition2.RenditionDefinition2;
import org.alfresco.repo.rendition2.RenditionService2Impl;
import org.alfresco.repo.rendition2.TransformClient;
import org.alfresco.repo.search.impl.noindex.NoIndexCategoryServiceImpl;
import org.alfresco.repo.security.authentication.AuthenticationComponent;
import org.alfresco.repo.tagging.TaggingServiceImpl;
import org.alfresco.repo.transaction.RetryingTransactionHelper;
import org.alfresco.service.cmr.dictionary.DictionaryService;
import org.alfresco.service.cmr.repository.ContentIOException;
@@ -48,8 +51,9 @@ import org.alfresco.service.cmr.repository.MimetypeService;
import org.alfresco.service.cmr.repository.NodeRef;
import org.alfresco.service.cmr.repository.NodeService;
import org.alfresco.service.cmr.repository.StoreRef;
import org.alfresco.service.cmr.tagging.TaggingService;
import org.alfresco.service.cmr.search.CategoryService;
import org.alfresco.service.namespace.NamespacePrefixResolver;
import org.alfresco.service.namespace.NamespaceService;
import org.alfresco.service.namespace.QName;
import org.alfresco.service.transaction.TransactionService;
import org.alfresco.test_category.BaseSpringTestsCategory;
@@ -67,15 +71,20 @@ import javax.transaction.NotSupportedException;
import javax.transaction.RollbackException;
import javax.transaction.SystemException;
import javax.transaction.UserTransaction;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.Serializable;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.StringJoiner;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -108,6 +117,7 @@ public class AsynchronousExtractorTest extends BaseSpringTest
private static final ExecutorService executorService = Executors.newCachedThreadPool();
private NodeService nodeService;
private NodeService publicNodeService;
private ContentService contentService;
private DictionaryService dictionaryService;
private MimetypeService mimetypeService;
@@ -120,11 +130,13 @@ public class AsynchronousExtractorTest extends BaseSpringTest
private TransformerDebug transformerDebug;
private TransactionService transactionService;
private TransformServiceRegistry transformServiceRegistry;
private TaggingService taggingService;
private TaggingServiceImpl taggingService;
private ContentMetadataExtracter contentMetadataExtracter;
private ContentMetadataEmbedder contentMetadataEmbedder;
private RenditionService2Impl renditionService2;
private TransformClient transformClient;
private CategoryService categoryService;
private MockCategoryService mockCategoryService;
private long origSize;
private Map<QName, Serializable> origProperties;
@@ -162,7 +174,7 @@ public class AsynchronousExtractorTest extends BaseSpringTest
* @param changedHashcode if specified indicates that the source node content changed or was deleted between
* the request to extract or embed and the response.
*/
TestAsynchronousExtractor(String mockResult, Integer changedHashcode)
TestAsynchronousExtractor(String mockResult, Integer changedHashcode, OverwritePolicy policy)
{
this.mockResult = mockResult;
this.changedHashcode = changedHashcode;
@@ -174,14 +186,17 @@ public class AsynchronousExtractorTest extends BaseSpringTest
setContentService(contentService);
setTransactionService(transactionService);
setTransformServiceRegistry(transformServiceRegistry);
setEnableStringTagging(true);
setTaggingService(taggingService);
setRegistry(metadataExtracterRegistry);
setMimetypeService(mimetypeService);
setDictionaryService(dictionaryService);
setDictionaryService(AsynchronousExtractorTest.this.dictionaryService);
setExecutorService(executorService);
setOverwritePolicy(policy);
register();
renditionService2.setTransformClient(mockTransformClient);
renditionService2.setAsynchronousExtractor(this);
}
@Override
@@ -261,10 +276,52 @@ public class AsynchronousExtractorTest extends BaseSpringTest
}
}
/**
* Mock CategoryService that only knows about taggable. Unlike the real implementation it does not call Solr to
* find tag nodes.
*/
private class MockCategoryService extends NoIndexCategoryServiceImpl
{
private NodeRef taggableCat;
MockCategoryService(NodeService nodeService, NodeService publicNodeService)
{
setNodeService(nodeService);
setPublicNodeService(publicNodeService);
// Create the required tagging category
NodeRef catContainer = nodeService.createNode(rootNodeRef,
ContentModel.ASSOC_CHILDREN, QName.createQName(NamespaceService.CONTENT_MODEL_1_0_URI,
"categoryContainer"), ContentModel.TYPE_CONTAINER).getChildRef();
NodeRef catRoot = nodeService.createNode(
catContainer,
ContentModel.ASSOC_CHILDREN,
QName.createQName(NamespaceService.CONTENT_MODEL_1_0_URI, "categoryRoot"),
ContentModel.TYPE_CATEGORYROOT).getChildRef();
taggableCat = nodeService.createNode(
catRoot,
ContentModel.ASSOC_CATEGORIES,
ContentModel.ASPECT_TAGGABLE,
ContentModel.TYPE_CATEGORY).getChildRef();
}
public void setTaggableCat(NodeRef taggableCat)
{
this.taggableCat = taggableCat;
}
@Override
protected Set<NodeRef> getClassificationNodes(StoreRef storeRef, QName qname)
{
return Collections.singleton(taggableCat);
}
}
@Before
public void before() throws Exception
{
nodeService = (NodeService) applicationContext.getBean("nodeService");
publicNodeService = (NodeService) applicationContext.getBean("NodeService");
contentService = (ContentService) applicationContext.getBean("contentService");
dictionaryService = (DictionaryService) applicationContext.getBean("dictionaryService");
mimetypeService = (MimetypeService) applicationContext.getBean("mimetypeService");
@@ -273,8 +330,10 @@ public class AsynchronousExtractorTest extends BaseSpringTest
renditionService2 = (RenditionService2Impl) applicationContext.getBean("renditionService2");
transactionService = (TransactionService) applicationContext.getBean("transactionService");
transformServiceRegistry = (TransformServiceRegistry) applicationContext.getBean("transformServiceRegistry");
taggingService = (TaggingService) applicationContext.getBean("taggingService");
taggingService = (TaggingServiceImpl) applicationContext.getBean("taggingService");
transformClient = (TransformClient) applicationContext.getBean("transformClient");
asynchronousExtractor = (AsynchronousExtractor) applicationContext.getBean("extractor.Asynchronous");
categoryService = (CategoryService) applicationContext.getBean("CategoryService");
// Create an empty metadata extractor registry, so that if we add one it will be used
metadataExtracterRegistry = new MetadataExtracterRegistry();
@@ -284,8 +343,10 @@ public class AsynchronousExtractorTest extends BaseSpringTest
contentMetadataExtracter.setContentService(contentService);
contentMetadataExtracter.setDictionaryService(dictionaryService);
contentMetadataExtracter.setMetadataExtracterRegistry(metadataExtracterRegistry);
contentMetadataExtracter.setTaggingService(taggingService);
contentMetadataExtracter.setApplicableTypes(new String[]{ContentModel.TYPE_CONTENT.toString()});
contentMetadataExtracter.setCarryAspectProperties(true);
contentMetadataExtracter.setEnableStringTagging(true);
contentMetadataEmbedder = new ContentMetadataEmbedder();
contentMetadataEmbedder.setNodeService(nodeService);
@@ -327,6 +388,9 @@ public class AsynchronousExtractorTest extends BaseSpringTest
origSize = getSize(nodeRef);
mockCategoryService = new MockCategoryService(nodeService, publicNodeService);
taggingService.setCategoryService(mockCategoryService);
return null;
}
});
@@ -336,15 +400,18 @@ public class AsynchronousExtractorTest extends BaseSpringTest
public void after() throws Exception
{
renditionService2.setTransformClient(transformClient);
renditionService2.setAsynchronousExtractor(asynchronousExtractor);
taggingService.setCategoryService(categoryService);
}
private void assertAsyncMetadataExecute(ActionExecuterAbstractBase executor, String mockResult,
Integer changedHashcode, long expectedSize,
Map<QName, Serializable> expectedProperties,
OverwritePolicy policy,
QName... ignoreProperties) throws Exception
{
TestAsynchronousExtractor extractor = new TestAsynchronousExtractor(mockResult, changedHashcode);
TestAsynchronousExtractor extractor = new TestAsynchronousExtractor(mockResult, changedHashcode, policy);
extractor.setEnableStringTagging(true);
executeAction(executor, extractor);
assertContentSize(nodeRef, origSize, AFTER_CALLING_EXECUTE);
assertProperties(nodeRef, origProperties, AFTER_CALLING_EXECUTE, ignoreProperties);
@@ -433,35 +500,35 @@ public class AsynchronousExtractorTest extends BaseSpringTest
expectedProperties.put(QName.createQName("cm:title", namespacePrefixResolver), "The quick brown fox jumps over the lazy dog");
assertAsyncMetadataExecute(contentMetadataExtracter, "quick/quick.html_metadata.json",
UNCHANGED_HASHCODE, origSize, expectedProperties);
UNCHANGED_HASHCODE, origSize, expectedProperties, OverwritePolicy.PRAGMATIC);
}
@Test
public void testExtractNodeDeleted() throws Exception
{
assertAsyncMetadataExecute(contentMetadataExtracter, "quick/quick.html_metadata.json",
SOURCE_HAS_NO_CONTENT, origSize, origProperties);
SOURCE_HAS_NO_CONTENT, origSize, origProperties, OverwritePolicy.PRAGMATIC);
}
@Test
public void testExtractContentChanged() throws Exception
{
assertAsyncMetadataExecute(contentMetadataExtracter, "quick/quick.html_metadata.json",
1234, origSize, origProperties);
CHANGED_HASHCODE, origSize, origProperties, OverwritePolicy.PRAGMATIC);
}
@Test
public void testExtractTransformFailure() throws Exception
{
assertAsyncMetadataExecute(contentMetadataExtracter, null,
UNCHANGED_HASHCODE, origSize, origProperties);
UNCHANGED_HASHCODE, origSize, origProperties, OverwritePolicy.PRAGMATIC);
}
@Test
public void testExtractTransformCorrupt() throws Exception
{
assertAsyncMetadataExecute(contentMetadataExtracter, "quick.html", // not json
UNCHANGED_HASHCODE, origSize, origProperties);
UNCHANGED_HASHCODE, origSize, origProperties, OverwritePolicy.PRAGMATIC);
}
@Test
@@ -471,7 +538,7 @@ public class AsynchronousExtractorTest extends BaseSpringTest
// "{http://www.unknown}name": "ignored" - is reported in an ERROR log
expectedProperties.put(QName.createQName("cm:author", namespacePrefixResolver), "Used");
assertAsyncMetadataExecute(contentMetadataExtracter, "quick/unknown_namespace_metadata.json",
UNCHANGED_HASHCODE, origSize, expectedProperties);
UNCHANGED_HASHCODE, origSize, expectedProperties, OverwritePolicy.PRAGMATIC);
}
@Test
@@ -488,7 +555,7 @@ public class AsynchronousExtractorTest extends BaseSpringTest
expectedProperties.put(QName.createQName("cm:originator", namespacePrefixResolver), "Mark Rogers");
assertAsyncMetadataExecute(contentMetadataExtracter, "quick/quick.msg_metadata.json",
UNCHANGED_HASHCODE, origSize, expectedProperties);
UNCHANGED_HASHCODE, origSize, expectedProperties, OverwritePolicy.PRAGMATIC);
Serializable sentDate = properties.get(QName.createQName("cm:sentdate", namespacePrefixResolver));
}
@@ -514,7 +581,7 @@ public class AsynchronousExtractorTest extends BaseSpringTest
// Note: As the metadata is for eml, an aspect gets added resulting in a second extract because of
// ImapContentPolicy.onAddAspect. I cannot see a good way to avoid this.
assertAsyncMetadataExecute(contentMetadataExtracter, "quick/quick.eml_metadata.json",
UNCHANGED_HASHCODE, origSize, expectedProperties,
UNCHANGED_HASHCODE, origSize, expectedProperties, OverwritePolicy.PRAGMATIC,
// cm:author is not in the quick.eml_metadata.json but is being added by the second extract which thinks
// the source mimetype is MimetypeMap.MIMETYPE_PDF, because that is what the before() method sets the
// content to. As a result the PdfBox metadata extractor is called, which extracts cm:author. Given that
@@ -523,37 +590,253 @@ public class AsynchronousExtractorTest extends BaseSpringTest
QName.createQName("cm:author", namespacePrefixResolver));
}
@Test
public void testEmbed() throws Exception
{
URL resource = getClass().getClassLoader().getResource("quick/quick.html");
assertNotNull("File not found", resource);
File file = new File(resource.toURI());
long fileSize = file.length();
assertAsyncMetadataExecute(contentMetadataEmbedder, "quick/quick.html", // just replace the pdf with html!
UNCHANGED_HASHCODE, 428, expectedProperties);
UNCHANGED_HASHCODE, fileSize, expectedProperties, OverwritePolicy.PRAGMATIC);
}
@Test
public void testEmbedNodeDeleted() throws Exception
{
assertAsyncMetadataExecute(contentMetadataEmbedder, "quick/quick.html",
SOURCE_HAS_NO_CONTENT, origSize, origProperties);
SOURCE_HAS_NO_CONTENT, origSize, origProperties, OverwritePolicy.PRAGMATIC);
}
@Test
public void testEmbedContentChanged() throws Exception
{
assertAsyncMetadataExecute(contentMetadataEmbedder, "quick/quick.html",
1234, origSize, origProperties);
CHANGED_HASHCODE, origSize, origProperties, OverwritePolicy.PRAGMATIC);
}
@Test
public void testEmbedTransformFailure() throws Exception
{
assertAsyncMetadataExecute(contentMetadataEmbedder, null,
UNCHANGED_HASHCODE, origSize, origProperties);
UNCHANGED_HASHCODE, origSize, origProperties, OverwritePolicy.PRAGMATIC);
}
// TODO Write tests for: overwritePolicy, enableStringTagging and carryAspectProperties.
// Values are set in AsynchronousExtractor.setMetadata(...) but make use of original code within
// MetadataExtracter and AbstractMappingMetadataExtracter.
// As the tests for exiting extractors are to be removed in ACS 7.0, it is possible that they were being used
// to test these values.
@Test
public void testOverwritePolicyEager() throws Exception
{
QName author = QName.createQName("cm:author", namespacePrefixResolver);
transactionService.getRetryingTransactionHelper().doInTransaction(() -> {
Map<QName, Serializable> properties = nodeService.getProperties(nodeRef);
properties.put(author, "Original author");
nodeService.addProperties(nodeRef, properties);
return null;
});
origProperties.put(author, "Original author");
expectedProperties.put(author, "Nevin Nollop");
assertAsyncMetadataExecute(contentMetadataExtracter, "quick/quick.eager_policy_metadata.json",
UNCHANGED_HASHCODE, origSize, expectedProperties, OverwritePolicy.EAGER);
}
@Test
public void testOverwritePolicyCautious() throws Exception
{
QName author = QName.createQName("cm:author", namespacePrefixResolver);
QName title = QName.createQName("cm:title", namespacePrefixResolver);
transactionService.getRetryingTransactionHelper().doInTransaction(() -> {
Map<QName, Serializable> properties = nodeService.getProperties(nodeRef);
properties.put(author, "Original author");
nodeService.addProperties(nodeRef, properties);
return null;
});
origProperties.put(author, "Original author");
expectedProperties.put(author, "Original author");
expectedProperties.put(title, "The quick brown fox jumps over the lazy dog");
assertAsyncMetadataExecute(contentMetadataExtracter, "quick/quick.cautious_policy_metadata.json",
UNCHANGED_HASHCODE, origSize, expectedProperties, OverwritePolicy.CAUTIOUS);
}
@Test
public void testOverwritePolicyPrudent() throws Exception
{
QName author = QName.createQName("cm:author", namespacePrefixResolver);
QName title = QName.createQName("cm:title", namespacePrefixResolver);
QName description = QName.createQName("cm:description", namespacePrefixResolver);
QName audio = QName.createQName("audio:audio", namespacePrefixResolver);
transactionService.getRetryingTransactionHelper().doInTransaction(() -> {
Map<QName, Serializable> properties = nodeService.getProperties(nodeRef);
properties.put(author, "Original author");
properties.put(title, ""); // will be changed
properties.put(audio, "Default audio"); // will not be changed because it's a media property
nodeService.addProperties(nodeRef, properties);
return null;
});
origProperties.put(author, "Original author");
origProperties.put(title, "");
origProperties.put(audio, "Default audio");
expectedProperties.put(author, "Original author");
expectedProperties.put(title, "The quick brown fox jumps over the lazy dog");
expectedProperties.put(description, "Gym class featuring a brown fox and lazy dog");
expectedProperties.put(audio, "Default audio");
assertAsyncMetadataExecute(contentMetadataExtracter, "quick/quick.prudent_policy_metadata.json",
UNCHANGED_HASHCODE, origSize, expectedProperties, OverwritePolicy.PRUDENT);
}
@Test
public void testOverwritePolicyPragmatic() throws Exception
{
QName author = QName.createQName("cm:author", namespacePrefixResolver);
QName title = QName.createQName("cm:title", namespacePrefixResolver);
QName description = QName.createQName("cm:description", namespacePrefixResolver);
QName audio = QName.createQName("audio:audio", namespacePrefixResolver);
transactionService.getRetryingTransactionHelper().doInTransaction(() -> {
Map<QName, Serializable> properties = nodeService.getProperties(nodeRef);
properties.put(author, "Original author");
properties.put(title, ""); // will be changed
properties.put(audio, "Default audio"); // will be changed using PRAGMATIC policy
nodeService.addProperties(nodeRef, properties);
return null;
});
origProperties.put(author, "Original author");
origProperties.put(title, "");
origProperties.put(audio, "Default audio");
expectedProperties.put(author, "Original author");
expectedProperties.put(title, "The quick brown fox jumps over the lazy dog");
expectedProperties.put(description, "Gym class featuring a brown fox and lazy dog");
expectedProperties.put(audio, "New audio");
assertAsyncMetadataExecute(contentMetadataExtracter, "quick/quick.pragmatic_policy_metadata.json",
UNCHANGED_HASHCODE, origSize, expectedProperties, OverwritePolicy.PRAGMATIC);
}
@Test
public void testCarryAspectFalse() throws Exception
{
QName title = QName.createQName("cm:title", namespacePrefixResolver);
QName author = QName.createQName("cm:author", namespacePrefixResolver);
QName description = QName.createQName("cm:description", namespacePrefixResolver);
transactionService.getRetryingTransactionHelper().doInTransaction(() -> {
Map<QName, Serializable> properties = nodeService.getProperties(nodeRef);
properties.put(title, "Default title");
nodeService.addProperties(nodeRef, properties);
return null;
});
origProperties.put(title, "Default title");
expectedProperties.put(author, "Nevin Nollop");
expectedProperties.put(description, "Gym class featuring a brown fox and lazy dog");
assertAsyncMetadataExecute(contentMetadataExtracter, "quick/quick.carryAspectFalse_metadata.json",
UNCHANGED_HASHCODE, origSize, expectedProperties, OverwritePolicy.PRAGMATIC);
transactionService.getRetryingTransactionHelper().doInTransaction(() -> {
Map<QName, Serializable> properties = nodeService.getProperties(nodeRef);
assertFalse("Title should be removed from properties because the extracted value is null",
properties.containsKey(title));
return null;
});
}
@Test
public void testCarryAspectTrue() throws Exception
{
QName title = QName.createQName("cm:title", namespacePrefixResolver);
QName author = QName.createQName("cm:author", namespacePrefixResolver);
QName description = QName.createQName("cm:description", namespacePrefixResolver);
transactionService.getRetryingTransactionHelper().doInTransaction(() -> {
Map<QName, Serializable> properties = nodeService.getProperties(nodeRef);
properties.put(title, "Default title");
nodeService.addProperties(nodeRef, properties);
return null;
});
origProperties.put(title, "Default title");
expectedProperties.put(author, "Nevin Nollop");
expectedProperties.put(title, "Default title");
expectedProperties.put(description, "Gym class featuring a brown fox and lazy dog");
assertAsyncMetadataExecute(contentMetadataExtracter, "quick/quick.carryAspectTrue_metadata.json",
UNCHANGED_HASHCODE, origSize, expectedProperties, OverwritePolicy.PRAGMATIC);
transactionService.getRetryingTransactionHelper().doInTransaction(() -> {
Map<QName, Serializable> properties = nodeService.getProperties(nodeRef);
assertTrue("Title should NOT be removed from properties because carryAspectProperties is true",
properties.containsKey(title));
return null;
});
}
@Test
public void testExtractTagging() throws Exception
{
QName taggable = QName.createQName("cm:taggable", namespacePrefixResolver);
contentMetadataExtracter.setEnableStringTagging(true);
transactionService.getRetryingTransactionHelper().doInTransaction(() -> {
Map<QName, Serializable> properties = nodeService.getProperties(nodeRef);
assertFalse(properties.containsKey(ContentModel.PROP_TAGS));
return null;});
List<String> expectedTags = Arrays.asList("tag1", "tag2", "tag3");
assertAsyncMetadataExecute(contentMetadataExtracter, "quick/quick.tagging_metadata.json",
UNCHANGED_HASHCODE, origSize, expectedProperties, OverwritePolicy.PRAGMATIC, taggable);
List<String> actualTags = transactionService.getRetryingTransactionHelper().doInTransaction(() -> {
Map<QName, Serializable> properties = nodeService.getProperties(nodeRef);
assertTrue(properties.containsKey(ContentModel.PROP_TAGS));
return taggingService.getTags(nodeRef);
});
for (String expectedTag : expectedTags)
{
assertTrue("Expected tag " + expectedTag + " not in " + actualTags, actualTags.contains(expectedTag));
}
}
@Test
public void testExtractTaggingWhenDisabled() throws Exception
{
contentMetadataExtracter.setEnableStringTagging(false);
transactionService.getRetryingTransactionHelper().doInTransaction(() -> {
Map<QName, Serializable> properties = nodeService.getProperties(nodeRef);
assertFalse(properties.containsKey(ContentModel.PROP_TAGS));
return null;
});
assertAsyncMetadataExecute(contentMetadataExtracter, "quick/quick.tagging_metadata_enable_false.json",
UNCHANGED_HASHCODE, origSize, expectedProperties, OverwritePolicy.PRAGMATIC);
List<String> tags = transactionService.getRetryingTransactionHelper().doInTransaction(() -> {
Map<QName, Serializable> properties = nodeService.getProperties(nodeRef);
assertFalse(properties.containsKey(ContentModel.PROP_TAGS));
return taggingService.getTags(nodeRef);
});
assertEquals("Unexpected tags", 0, tags.size());
}
}