ACS-960 Disable very frequent intermittent test failures (#187)

Part of Epic: ACS-959 Fix or suppress the content repository's intermittent test failures

core/src/test/java/org/alfresco/util/testing/category/FrequentlyFailingTests.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright (C) 2005-2020 Alfresco Software Limited.
+ *
+ * This file is part of Alfresco
+ *
+ * Alfresco is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Alfresco is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.alfresco.util.testing.category;
+
+/**
+ * Category marker for tests that fail very frequently so are excluded from the build. Only run by developers while
+ * working on a specific feature/class. An issue should be raised to identify and fix the reason the test fails.
+ * @see IntermittentlyFailingTests for tests that fail intermittently but not that frequently
+ */
+public interface FrequentlyFailingTests extends NonBuildTests
+{
+}

core/src/test/java/org/alfresco/util/testing/category/IntermittentlyFailingTests.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright (C) 2005-2020 Alfresco Software Limited.
+ *
+ * This file is part of Alfresco
+ *
+ * Alfresco is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Alfresco is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.alfresco.util.testing.category;
+
+/**
+ * Category marker for tests that intermittently fail, but not that frequently. Still included in the build. If the
+ * instability increases or we have time to investigate, an issue should be raised to investigate and possibly
+ * be switch to {@link FrequentlyFailingTests}. Use by developers to aid in working out if the build job should just be
+ * retried.
+ */
+public interface IntermittentlyFailingTests
+{
+}

packaging/tests/tas-cmis/src/test/java/org/alfresco/cmis/DeleteTreeTests.java

@@ -149,6 +149,7 @@ public class DeleteTreeTests extends CmisTest
                 .createFile(testFile).and().assertThat().existsInRepo()
                 .and().checkOut()
             .when().usingResource(parentTestFolder).deleteFolderTree(true, UnfileObject.DELETESINGLEFILED, true)
+                /* @Category(IntermittentlyFailingTests.class) ACS-959 Intermittent failure on next line. @Category not supported by TAS tests. */
                 .and().assertThat().doesNotExistInRepo()
             .then().usingResource(childTestFolder).assertThat().doesNotExistInRepo()
                    .usingResource(testFile).assertThat().doesNotExistInRepo();

packaging/tests/tas-cmis/src/test/java/org/alfresco/cmis/GetChildrenTests.java

@@ -64,6 +64,7 @@ public class GetChildrenTests extends CmisTest
     @Test(groups = { TestGroup.REGRESSION, TestGroup.CMIS} , expectedExceptions = CmisObjectNotFoundException.class)
     public void getChildrenFromDeletedFolder() throws Exception
     {
+        /* @Category(IntermittentlyFailingTests.class) ACS-959 Intermittent failure. @Category not supported by TAS tests. */
         cmisApi.authenticateUser(testUser).usingSite(testSite)
             .createFolder(testFolder)
             .and().usingResource(testFolder)

packaging/tests/tas-integration/src/test/java/org/alfresco/tas/integration/IntegrationFullTests.java

@@ -86,6 +86,7 @@ public class IntegrationFullTests extends IntegrationTest
         SiteModel testSite = dataSite.usingUser(testUser).createIMAPSite();
         FileModel testFile = dataContent.usingUser(testUser).usingSite(testSite).createContent(FileModel.getRandomFileModel(FileType.TEXT_PLAIN));
         imapProtocol.authenticateUser(testUser).usingSite(testSite).usingResource(testFile).deleteMessage();
+        /* @Category(IntermittentlyFailingTests.class) ACS-959 Intermittent failure on next line. @Category not supported by TAS tests. */
         cmisAPI.authenticateUser(testUser).usingSite(testSite).usingResource(testFile).rename("new file name");
     }
 

packaging/tests/tas-integration/src/test/java/org/alfresco/tas/integration/IntegrationFullTests1.java

@@ -28,6 +28,7 @@ import org.alfresco.utility.model.TestGroup;
 import org.alfresco.utility.model.UserModel;
 import org.alfresco.utility.testrail.ExecutionType;
 import org.alfresco.utility.testrail.annotation.TestRail;
+import org.junit.experimental.categories.Category;
 import org.springframework.http.HttpMethod;
 import org.springframework.http.HttpStatus;
 import org.testng.Assert;
@@ -458,6 +459,8 @@ public class IntegrationFullTests1 extends IntegrationTest
         ftpProtocol.authenticateUser(testUser).usingSite(testSitePublic).usingResource(file1).update(newContent)   
             .assertThat().contentIs(newContent);
         
+        /* @Category(FrequentlyFailingTests.class) ACS-965 - @Category not supported by TAS tests, so commenting out for now
+
         STEP("8. U1 adds tags to file1 using RestAPI");
         String newTag = RandomData.getRandomName("tag");
         RestTagModel tag = restAPI.authenticateUser(testUser).withCoreAPI().usingResource(file1).addTag(newTag);
@@ -473,5 +476,7 @@ public class IntegrationFullTests1 extends IntegrationTest
             .assertThat().contentPropertyHasValue("cmis:name", newName)
             .assertThat().contentPropertyHasValue("cm:taggable", tag.getId())
             .assertThat().contentIs(newContent);
+
+        */
     }
 }

packaging/tests/tas-restapi/src/test/java/org/alfresco/rest/comments/AddCommentsTests.java

@@ -242,7 +242,8 @@ public class AddCommentsTests extends RestTest
         restClient.assertStatusCodeIs(HttpStatus.METHOD_NOT_ALLOWED).assertLastError().containsSummary(RestErrorModel.CANNOT_COMMENT);
     }
     
-    @TestRail(section = { TestGroup.REST_API, TestGroup.COMMENTS }, executionType = ExecutionType.REGRESSION, 
+    /* @Category(FrequentlyFailingTests.class) ACS-968 - @Category not supported by TAS tests, so commenting out for now
+    @TestRail(section = { TestGroup.REST_API, TestGroup.COMMENTS }, executionType = ExecutionType.REGRESSION,
             description = "Verify that comments cannot be added to a tag")
     @Test(groups = { TestGroup.REST_API, TestGroup.COMMENTS, TestGroup.REGRESSION })
     public void addCommentsToATag() throws JsonToModelConversionException, Exception
@@ -257,6 +258,7 @@ public class AddCommentsTests extends RestTest
         
         restClient.assertStatusCodeIs(HttpStatus.METHOD_NOT_ALLOWED).assertLastError().containsSummary(RestErrorModel.CANNOT_COMMENT);
     }
+    */
 
     @TestRail(section={TestGroup.REST_API, TestGroup.COMMENTS}, executionType= ExecutionType.REGRESSION,
             description= "Using Manager user verify that you can provide a large string for one comment")

remote-api/src/test/java/org/alfresco/repo/web/scripts/servlet/RemoteAuthenticatorFactoryAdminConsoleAccessTest.java

@@ -2,7 +2,7 @@
  * #%L
  * Alfresco Repository
  * %%
- * Copyright (C) 2005 - 2018 Alfresco Software Limited
+ * Copyright (C) 2005 - 2020 Alfresco Software Limited
  * %%
  * This file is part of the Alfresco software.
  * If the software was purchased under a paid Alfresco license, the terms of
@@ -37,12 +37,15 @@ import org.alfresco.service.cmr.security.MutableAuthenticationService;
 import org.alfresco.service.cmr.security.PersonService;
 import org.alfresco.util.BaseSpringTest;
 import org.alfresco.util.PropertyMap;
+import org.alfresco.util.testing.category.FrequentlyFailingTests;
+import org.alfresco.util.testing.category.IntermittentlyFailingTests;
 import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.junit.Before;
 import org.junit.Test;
+import org.junit.experimental.categories.Category;
 import org.junit.runner.RunWith;
 import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
@@ -204,6 +207,7 @@ public class RemoteAuthenticatorFactoryAdminConsoleAccessTest extends BaseSpring
         complexCheckOfScriptCases(families);
     }
 
+    @Category(IntermittentlyFailingTests.class) // ACS-959
     @Test
     public void testAdminCanAccessAdminConsoleHelperScript()
     {

remote-api/src/test/java/org/alfresco/repo/webdav/WebDAVMethodTest.java

@@ -2,7 +2,7 @@
  * #%L
  * Alfresco Remote API
  * %%
- * Copyright (C) 2005 - 2016 Alfresco Software Limited
+ * Copyright (C) 2005 - 2020 Alfresco Software Limited
  * %%
  * This file is part of the Alfresco software. 
  * If the software was purchased under a paid Alfresco license, the terms of 
@@ -48,11 +48,13 @@ import org.alfresco.service.namespace.QName;
 import org.alfresco.service.transaction.TransactionService;
 import org.alfresco.util.ApplicationContextHelper;
 import org.alfresco.util.GUID;
+import org.alfresco.util.testing.category.IntermittentlyFailingTests;
 import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.junit.Test;
+import org.junit.experimental.categories.Category;
 import org.junit.runner.RunWith;
 import org.mockito.Mock;
 import org.mockito.runners.MockitoJUnitRunner;
@@ -404,6 +406,7 @@ public class WebDAVMethodTest
     }
     
     /* MNT-10555 Test */
+    @Category(IntermittentlyFailingTests.class) // ACS-959
     @Test
     public void expiryLockTest()
     {

remote-api/src/test/java/org/alfresco/rest/DeletedNodesTest.java

@@ -2,7 +2,7 @@
  * #%L
  * Alfresco Remote API
  * %%
- * Copyright (C) 2005 - 2016 Alfresco Software Limited
+ * Copyright (C) 2005 - 2020 Alfresco Software Limited
  * %%
  * This file is part of the Alfresco software. 
  * If the software was purchased under a paid Alfresco license, the terms of 
@@ -58,8 +58,10 @@ import org.alfresco.rest.api.tests.client.data.Rendition;
 import org.alfresco.rest.api.tests.util.MultiPartBuilder;
 import org.alfresco.rest.api.tests.util.RestApiUtil;
 import org.alfresco.rest.api.trashcan.TrashcanEntityResource;
+import org.alfresco.util.testing.category.IntermittentlyFailingTests;
 import org.junit.After;
 import org.junit.Test;
+import org.junit.experimental.categories.Category;
 import org.springframework.extensions.webscripts.Status;
 
 import com.google.common.collect.Ordering;
@@ -329,6 +331,7 @@ public class DeletedNodesTest extends AbstractSingleNetworkSiteTest
      * <p>GET:</p>
      * {@literal <host>:<port>/alfresco/api/-default-/public/alfresco/versions/1/deleted-nodes/<nodeId>/content}
      */
+    @Category(IntermittentlyFailingTests.class) // ACS-959
     @Test
     public void testDownloadFileContent() throws Exception
     {

remote-api/src/test/java/org/alfresco/rest/api/tests/AbstractEnterpriseOpenCMIS11TCKTest.java

@@ -2,7 +2,7 @@
  * #%L
  * Alfresco Remote API
  * %%
- * Copyright (C) 2005 - 2016 Alfresco Software Limited
+ * Copyright (C) 2005 - 2020 Alfresco Software Limited
  * %%
  * This file is part of the Alfresco software. 
  * If the software was purchased under a paid Alfresco license, the terms of 
@@ -30,6 +30,7 @@ import java.util.Map;
 import org.alfresco.opencmis.tck.tests.query.QueryForObjectCustom;
 import org.alfresco.opencmis.tck.tests.query.QueryInFolderTestCustom;
 import org.alfresco.opencmis.tck.tests.query.QueryLikeTestCustom;
+import org.alfresco.util.testing.category.IntermittentlyFailingTests;
 import org.alfresco.util.testing.category.LuceneTests;
 import org.alfresco.util.testing.category.RedundantTests;
 import org.apache.chemistry.opencmis.tck.impl.AbstractSessionTestGroup;
@@ -64,6 +65,7 @@ import org.junit.experimental.categories.Category;
 
 public abstract class AbstractEnterpriseOpenCMIS11TCKTest extends AbstractEnterpriseOpenCMISTCKTest
 {
+    @Category(IntermittentlyFailingTests.class) // ACS-959
     @Test
     public void testCMISTCKBasics() throws Exception
     {

remote-api/src/test/java/org/alfresco/rest/api/tests/SharedLinkApiTest.java

@@ -2,7 +2,7 @@
  * #%L
  * Alfresco Remote API
  * %%
- * Copyright (C) 2005 - 2016 Alfresco Software Limited
+ * Copyright (C) 2005 - 2020 Alfresco Software Limited
  * %%
  * This file is part of the Alfresco software. 
  * If the software was purchased under a paid Alfresco license, the terms of 
@@ -65,6 +65,7 @@ import org.alfresco.service.cmr.search.SearchService;
 import org.alfresco.service.cmr.security.AccessStatus;
 import org.alfresco.service.cmr.security.PermissionService;
 import org.alfresco.service.cmr.site.SiteVisibility;
+import org.alfresco.util.testing.category.FrequentlyFailingTests;
 import org.alfresco.util.testing.category.LuceneTests;
 import org.alfresco.util.testing.category.RedundantTests;
 import org.joda.time.DateTime;

repository/src/test/java/org/alfresco/encryption/EncryptionTests.java

@@ -2,7 +2,7 @@
  * #%L
  * Alfresco Repository
  * %%
- * Copyright (C) 2005 - 2016 Alfresco Software Limited
+ * Copyright (C) 2005 - 2020 Alfresco Software Limited
  * %%
  * This file is part of the Alfresco software. 
  * If the software was purchased under a paid Alfresco license, the terms of 
@@ -59,6 +59,8 @@ import org.alfresco.service.namespace.QName;
 import org.alfresco.service.transaction.TransactionService;
 import org.alfresco.util.ApplicationContextHelper;
 import org.alfresco.util.Pair;
+import org.alfresco.util.testing.category.IntermittentlyFailingTests;
+import org.junit.experimental.categories.Category;
 import org.springframework.context.ApplicationContext;
 
 public class EncryptionTests extends TestCase
@@ -335,7 +337,8 @@ public class EncryptionTests extends TestCase
 	{
 		testChangeKeysImpl(true);
 	}
-	
+
+	@Category(IntermittentlyFailingTests.class) // ACS-959
 	public void testFailedEncryptionWithCachedCiphers() throws Throwable
 	{
 		Pair<byte[], AlgorithmParameters> pair = null;

repository/src/test/java/org/alfresco/opencmis/CMISTest.java

@@ -3,7 +3,7 @@
  * #%L
  * Alfresco Repository
  * %%
- * Copyright (C) 2005 - 2016 Alfresco Software Limited
+ * Copyright (C) 2005 - 2020 Alfresco Software Limited
  * %%
  * This file is part of the Alfresco software. 
  * If the software was purchased under a paid Alfresco license, the terms of 
@@ -119,7 +119,9 @@ import org.alfresco.service.namespace.QName;
 import org.alfresco.service.transaction.TransactionService;
 import org.alfresco.util.ApplicationContextHelper;
 import org.alfresco.util.Pair;
+import org.alfresco.util.testing.category.FrequentlyFailingTests;
 import org.alfresco.util.testing.category.LuceneTests;
+import org.alfresco.util.testing.category.PerformanceTests;
 import org.alfresco.util.testing.category.RedundantTests;
 import org.apache.chemistry.opencmis.commons.PropertyIds;
 import org.apache.chemistry.opencmis.commons.data.Ace;
@@ -744,6 +746,7 @@ public class CMISTest
     /**
      * ALF-18006 Test content mimetype auto-detection into CmisStreamInterceptor when "Content-Type" is not defined.
      */
+    @Category(FrequentlyFailingTests.class) // ACS-962
     @Test
     public void testContentMimeTypeDetection()
     {
@@ -3952,6 +3955,7 @@ public class CMISTest
      *  This test makes sure that once a copy is checked out, updateProperties method can be called
      *  and properly adds the new properties.
      */
+    @Category(FrequentlyFailingTests.class) // ACS-961
     @Test
     public void aPrivateCopyMustAllowTheAdditionOfAspects_CMIS_1_1_Version()
     {

repository/src/test/java/org/alfresco/opencmis/OpenCmisLocalTest.java

@@ -2,7 +2,7 @@
  * #%L
  * Alfresco Repository
  * %%
- * Copyright (C) 2005 - 2016 Alfresco Software Limited
+ * Copyright (C) 2005 - 2020 Alfresco Software Limited
  * %%
  * This file is part of the Alfresco software. 
  * If the software was purchased under a paid Alfresco license, the terms of 
@@ -61,6 +61,7 @@ import org.alfresco.util.ApplicationContextHelper;
 import org.alfresco.util.FileFilterMode.Client;
 import org.alfresco.util.GUID;
 import org.alfresco.util.TempFileProvider;
+import org.alfresco.util.testing.category.FrequentlyFailingTests;
 import org.alfresco.util.testing.category.LuceneTests;
 import org.aopalliance.intercept.MethodInterceptor;
 import org.aopalliance.intercept.MethodInvocation;
@@ -466,6 +467,7 @@ public class OpenCmisLocalTest extends TestCase
      * This test would have fit better within CheckOutCheckInServiceImplTest but
      * was added here to make use of existing methods
      */
+    @Category(FrequentlyFailingTests.class) // ACS-962
     public void testCancelCheckoutWhileInCheckedOutState()
     {
         ServiceRegistry serviceRegistry = (ServiceRegistry) ctx.getBean(ServiceRegistry.SERVICE_REGISTRY);

repository/src/test/java/org/alfresco/repo/event2/ChildAssociationRepoEventIT.java

@@ -39,7 +39,9 @@ import org.alfresco.service.cmr.repository.ChildAssociationRef;
 import org.alfresco.service.cmr.repository.NodeRef;
 import org.alfresco.service.namespace.QName;
 import org.alfresco.util.GUID;
+import org.alfresco.util.testing.category.IntermittentlyFailingTests;
 import org.junit.Test;
+import org.junit.experimental.categories.Category;
 
 /**
  * @author Adina Ababei
@@ -346,6 +348,7 @@ public class ChildAssociationRepoEventIT extends AbstractContextAwareRepoEvent
         assertEquals("cm:contains", getChildAssocResource(childAssocEvents.get(2)).getAssocType());
     }
 
+    @Category(IntermittentlyFailingTests.class) // ACS-959
     @Test
     public void testOneParentMultipleChildrenSameTransaction()
     {

repository/src/test/java/org/alfresco/repo/transfer/NodeCrawlerTest.java

@@ -2,7 +2,7 @@
  * #%L
  * Alfresco Repository
  * %%
- * Copyright (C) 2005 - 2016 Alfresco Software Limited
+ * Copyright (C) 2005 - 2020 Alfresco Software Limited
  * %%
  * This file is part of the Alfresco software. 
  * If the software was purchased under a paid Alfresco license, the terms of 
@@ -49,6 +49,7 @@ import org.alfresco.test_category.BaseSpringTestsCategory;
 import org.alfresco.test_category.OwnJVMTestsCategory;
 import org.alfresco.util.BaseAlfrescoSpringTest;
 import org.alfresco.util.GUID;
+import org.alfresco.util.testing.category.IntermittentlyFailingTests;
 import org.alfresco.util.testing.category.LuceneTests;
 import org.junit.Before;
 import org.junit.Test;
@@ -162,6 +163,7 @@ public class NodeCrawlerTest extends BaseAlfrescoSpringTest
         assertEquals(node15, new ArrayList<NodeRef>(results).get(0));
     }
 
+    @Category(IntermittentlyFailingTests.class) // ACS-959
     @Test
     public void testCrawler()
     {