REPO-4329 Configure Custom renditions (#554)

* REPO-4329 Configure Custom renditions

- read external directories for pipelines and renditions
- bug fix: the renditions schedule was being cleared when the pipeline schedule was cleared

src/main/java/org/alfresco/repo/rendition2/RenditionDefinitionRegistry2Impl.java

@@ -57,12 +57,18 @@ public class RenditionDefinitionRegistry2Impl implements RenditionDefinitionRegi
     {
         Map<String, RenditionDefinition2> renditionDefinitions = new HashMap();
         Map<String, Set<Pair<String, Long>>> renditionsFor = new HashMap<>();
-        private int count = 0;
+        private int fileCount;
 
         @Override
         public String toString()
         {
-            return "("+count+")";
+            int renditionCount = renditionDefinitions.size();
+            return "(renditions: "+renditionCount+" files: "+fileCount+")";
+        }
+
+        public void setFileCount(int fileCount)
+        {
+            this.fileCount = fileCount;
         }
     }
 
@@ -126,6 +132,10 @@ public class RenditionDefinitionRegistry2Impl implements RenditionDefinitionRegi
         @Override
         public boolean readConfig() throws IOException
         {
+            if (configFileFinder != null)
+            {
+                configFileFinder.setFileCount(0);
+            }
             return RenditionDefinitionRegistry2Impl.this.readConfig();
         }
 
@@ -270,6 +280,7 @@ public class RenditionDefinitionRegistry2Impl implements RenditionDefinitionRegi
         }
         Data data = getData();
         data.renditionDefinitions.put(renditionName, renditionDefinition);
+        data.setFileCount(configFileFinder == null ? 0 : configFileFinder.getFileCount());
     }
 
     public void unregister(String renditionName)

src/main/java/org/alfresco/transform/client/model/config/CombinedConfig.java

@@ -74,6 +74,7 @@ public class CombinedConfig
     private List<TransformNodeAndItsOrigin> allTransforms = new ArrayList<>();
     private ObjectMapper jsonObjectMapper = new ObjectMapper();
     private ConfigFileFinder configFileFinder;
+    private int tEngineCount;
 
     static class TransformNodeAndItsOrigin
     {
@@ -149,7 +150,11 @@ public class CombinedConfig
         boolean successReadingConfig = true;
         for (String url : urls)
         {
-            if (!addRemoteConfig(url, remoteType))
+            if (addRemoteConfig(url, remoteType))
+            {
+                tEngineCount++ ;
+            }
+            else
             {
                 successReadingConfig = false;
             }
@@ -268,6 +273,9 @@ public class CombinedConfig
 
     public void register(TransformServiceRegistryImpl registry) throws IOException
     {
+        TransformServiceRegistryImpl.Data data = registry.getData();
+        data.setTEngineCount(tEngineCount);
+        data.setFileCount(configFileFinder.getFileCount());
         List<TransformAndItsOrigin> transformers = getTransforms();
         transformers.forEach(t->registry.register(t.transform, t.baseUrl, t.readFrom));
     }

src/main/java/org/alfresco/transform/client/model/config/TransformServiceRegistryImpl.java

@@ -31,7 +31,6 @@ import org.alfresco.util.ConfigScheduler;
 import org.alfresco.util.PropertyCheck;
 import org.apache.commons.logging.Log;
 import org.quartz.CronExpression;
-import org.quartz.SchedulerException;
 import org.springframework.beans.factory.InitializingBean;
 
 import java.io.IOException;
@@ -39,14 +38,11 @@ import java.io.Reader;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
-import org.springframework.http.converter.json.Jackson2ObjectMapperFactoryBean;
 
 import static org.alfresco.repo.rendition2.RenditionDefinition2.TIMEOUT;
 
@@ -61,12 +57,26 @@ public abstract class TransformServiceRegistryImpl implements TransformServiceRe
         ConcurrentMap<String, ConcurrentMap<String, List<SupportedTransform>>> cachedSupportedTransformList = new ConcurrentHashMap<>();
         private int transformerCount = 0;
         private int transformCount = 0;
+        private int tEngineCount = 0;
+        private int fileCount;
         boolean firstTime = true;
 
         @Override
         public String toString()
         {
-            return "("+transformerCount+":"+transformCount+")";
+            return transformerCount == 0 && transformCount == 0 && tEngineCount == 0 && fileCount == 0
+                    ? ""
+                    : "(transformers: "+transformerCount+" transforms: "+transformCount+" t-engines: "+tEngineCount+" files: "+fileCount+")";
+        }
+
+        public void setTEngineCount(int tEngineCount)
+        {
+            this.tEngineCount = tEngineCount;
+        }
+
+        public void setFileCount(int fileCount)
+        {
+            this.fileCount = fileCount;
         }
     }
 

src/main/java/org/alfresco/util/ConfigFileFinder.java

@@ -31,6 +31,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.commons.logging.Log;
 
 import java.io.File;
+import java.io.FileNotFoundException;
 import java.io.FileReader;
 import java.io.IOException;
 import java.io.InputStream;
@@ -42,6 +43,7 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.Enumeration;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.jar.JarEntry;
 import java.util.jar.JarFile;
 
@@ -54,67 +56,54 @@ import java.util.jar.JarFile;
 public abstract class ConfigFileFinder
 {
     private final ObjectMapper jsonObjectMapper;
+    private int fileCount;
 
     public ConfigFileFinder(ObjectMapper jsonObjectMapper)
     {
         this.jsonObjectMapper = jsonObjectMapper;
     }
 
+    public int getFileCount()
+    {
+        return fileCount;
+    }
+
+    public void setFileCount(int fileCount)
+    {
+        this.fileCount = fileCount;
+    }
+
     public boolean readFiles(String path, Log log)
     {
-        boolean successReadingConfig = true;
+        AtomicBoolean successReadingConfig = new AtomicBoolean(true);
         try
         {
-            boolean somethingRead = false;
+            AtomicBoolean somethingRead = new AtomicBoolean(false);
+
+            // Try reading resources in a jar
             final File jarFile = new File(getClass().getProtectionDomain().getCodeSource().getLocation().getPath());
             if (jarFile.isFile())
             {
-                Enumeration<JarEntry> entries = new JarFile(jarFile).entries(); // gives ALL entries in jar
+                readFromJar(jarFile, path, log, successReadingConfig, somethingRead);
-                String prefix = path + "/";
-                List<String> names = new ArrayList<>();
-                while (entries.hasMoreElements())
-                {
-                    final String name = entries.nextElement().getName();
-                    if (name.startsWith(prefix) && name.length() > prefix.length())
-                    {
-                        names.add(name);
-                    }
-                }
-                Collections.sort(names);
-                for (String name : names)
-                {
-                    somethingRead = true;
-                    successReadingConfig &= readFile(new InputStreamReader(getResourceAsStream(name)),"resource", name, null, log);
-                }
-
-                new JarFile(jarFile).close();
             }
             else
             {
+                // Try reading resources from disk
                 URL url = getClass().getClassLoader().getResource(path);
                 if (url != null)
                 {
-                    File root = new File(url.getPath());
+                    String urlPath = url.getPath();
-                    String rootPath = root.getPath();
+                    readFromDisk(urlPath, log, successReadingConfig, somethingRead);
-                    if (root.isDirectory())
-                    {
-                        File[] files = root.listFiles();
-                        Arrays.sort(files, (file1, file2) -> file1.getName().compareTo(file2.getName()));
-                        for (File file: files)
-                        {
-                            somethingRead = true;
-                            successReadingConfig &= readFile(new FileReader(file), "file", file.getPath(), null, log);
-                        }
-                    }
-                    else
-                    {
-                        somethingRead = true;
-                        successReadingConfig = readFile(new FileReader(root), "file", rootPath, null, log);
-                    }
                 }
             }
 
-            if (!somethingRead)
+            if (!somethingRead.get() && new File(path).exists())
+            {
+                // Try reading files from disk
+                readFromDisk(path, log, successReadingConfig, somethingRead);
+            }
+
+            if (!somethingRead.get())
             {
                 log.warn("No config read from "+path);
             }
@@ -122,9 +111,61 @@ public abstract class ConfigFileFinder
         catch (IOException e)
         {
             log.error("Error reading from "+path);
-            successReadingConfig = false;
+            successReadingConfig.set(false);
+        }
+        return successReadingConfig.get();
+    }
+
+    private void readFromJar(File jarFile, String path, Log log, AtomicBoolean successReadingConfig, AtomicBoolean somethingRead) throws IOException
+    {
+        JarFile jar = new JarFile(jarFile);
+        try
+        {
+            Enumeration<JarEntry> entries = jar.entries(); // gives ALL entries in jar
+            String prefix = path + "/";
+            List<String> names = new ArrayList<>();
+            while (entries.hasMoreElements())
+            {
+                final String name = entries.nextElement().getName();
+                if ((name.startsWith(prefix) && name.length() > prefix.length()) ||
+                    (name.equals(path)))
+                {
+                    names.add(name);
+                }
+            }
+            Collections.sort(names);
+            for (String name : names)
+            {
+                InputStreamReader reader = new InputStreamReader(getResourceAsStream(name));
+                readFromReader(successReadingConfig, somethingRead, reader, "resource", name, null, log);
+            }
+        }
+        finally
+        {
+            jar.close();
+        }
+    }
+
+    private void readFromDisk(String path, Log log, AtomicBoolean successReadingConfig, AtomicBoolean somethingRead) throws FileNotFoundException
+    {
+        File root = new File(path);
+        if (root.isDirectory())
+        {
+            File[] files = root.listFiles();
+            Arrays.sort(files, (file1, file2) -> file1.getName().compareTo(file2.getName()));
+            for (File file : files)
+            {
+                FileReader reader = new FileReader(file);
+                String filePath = file.getPath();
+                readFromReader(successReadingConfig, somethingRead, reader, "file", filePath, null, log);
+            }
+        }
+        else
+        {
+            FileReader reader = new FileReader(root);
+            String filePath = root.getPath();
+            readFromReader(successReadingConfig, somethingRead, reader, "file", filePath, null, log);
         }
-        return successReadingConfig;
     }
 
     private InputStream getResourceAsStream(String resource)
@@ -133,6 +174,20 @@ public abstract class ConfigFileFinder
         return in == null ? getClass().getResourceAsStream(resource) : in;
     }
 
+    private void readFromReader(AtomicBoolean successReadingConfig, AtomicBoolean somethingRead,
+                               Reader reader, String readFrom, String path, String baseUrl, Log log)
+    {
+        somethingRead.set(true);
+        boolean success = readFile(reader, readFrom, path, null, log);
+        if (success)
+        {
+            fileCount++;
+        }
+        boolean newSuccessReadingConfig = successReadingConfig.get();
+        newSuccessReadingConfig &= success;
+        successReadingConfig.set(newSuccessReadingConfig);
+    }
+
     public boolean readFile(Reader reader, String readFrom, String path, String baseUrl, Log log)
     {
         // At the moment it is assumed the file is Json, but that does not need to be the case.

src/main/java/org/alfresco/util/ConfigScheduler.java

@@ -36,6 +36,7 @@ import org.quartz.JobDataMap;
 import org.quartz.JobDetail;
 import org.quartz.JobExecutionContext;
 import org.quartz.JobExecutionException;
+import org.quartz.JobKey;
 import org.quartz.Scheduler;
 import org.quartz.TriggerBuilder;
 import org.quartz.impl.StdSchedulerFactory;
@@ -60,7 +61,7 @@ public abstract class ConfigScheduler<Data>
         {
             JobDataMap dataMap = context.getJobDetail().getJobDataMap();
             ConfigScheduler configScheduler = (ConfigScheduler)dataMap.get(CONFIG_SCHEDULER);
-            boolean successReadingConfig = configScheduler.readConfigAndReplace();
+            boolean successReadingConfig = configScheduler.readConfigAndReplace(true);
             configScheduler.changeScheduleOnStateChange(successReadingConfig);
         }
     }
@@ -76,6 +77,7 @@ public abstract class ConfigScheduler<Data>
     private CronExpression initialAndOnErrorCronExpression;
 
     private Scheduler scheduler;
+    private JobKey jobKey;
     private boolean normalCronSchedule;
 
     protected Data data;
@@ -140,7 +142,7 @@ public abstract class ConfigScheduler<Data>
             }
             else
             {
-                readConfigAndReplace();
+                readConfigAndReplace(false);
             }
         }
     }
@@ -155,6 +157,7 @@ public abstract class ConfigScheduler<Data>
                     .withIdentity(jobName)
                     .ofType(ConfigSchedulerJob.class)
                     .build();
+            jobKey = job.getKey();
             job.getJobDataMap().put(CONFIG_SCHEDULER, this);
             CronExpression cronExpression = normalCronSchedule ? this.cronExpression : initialAndOnErrorCronExpression;
             CronTrigger trigger = TriggerBuilder.newTrigger()
@@ -163,6 +166,7 @@ public abstract class ConfigScheduler<Data>
                     .build();
             scheduler.startDelayed(0);
             scheduler.scheduleJob(job, trigger);
+            log.debug("Schedule set "+cronExpression);
         }
         catch (Exception e)
         {
@@ -176,7 +180,9 @@ public abstract class ConfigScheduler<Data>
         {
             try
             {
-                scheduler.clear();
+                scheduler.deleteJob(jobKey);
+                scheduler = null;
+                jobKey = null;
             }
             catch (Exception e)
             {
@@ -185,10 +191,10 @@ public abstract class ConfigScheduler<Data>
         }
     }
 
-    private boolean readConfigAndReplace()
+    private boolean readConfigAndReplace(boolean scheduledRead)
     {
         boolean successReadingConfig;
-        log.debug("Config read started");
+        log.debug((scheduledRead ? "Scheduled" : "Unscheduled")+" config read started");
         Data data = getData();
         try
         {
@@ -197,7 +203,7 @@ public abstract class ConfigScheduler<Data>
             successReadingConfig = readConfig();
             data = newData;
             log.debug("Config read finished "+data+
-                    (successReadingConfig ? "" : ". Config replaced but there were problems"));
+                    (successReadingConfig ? "" : ". Config replaced but there were problems")+"\n");
         }
         catch (Exception e)
         {

src/main/resources/alfresco/repository.properties

@@ -1079,27 +1079,37 @@ people.search.honor.hint.useCQ=true
 # Delays cron jobs after bootstrap to allow server to fully come up before jobs start
 system.cronJob.startDelayMilliseconds=60000
 
-# Schedule for reading rendition config definitions dynamically. Initially checks every 10 seconds and then switches to
+# Schedule for reading mimetype config definitions dynamically. Initially checks every 10 seconds and then switches to
-# every 10 minutes after the configuration is read successfully. If there is a error later reading the config, the
+# every hour after the configuration is read successfully. If there is a error later reading the config, the
 # checks return to every 10 seconds.
-rendition.config.cronExpression=0 0/10 * * * ?
+mimetype.config.cronExpression=0 30 0/1 * * ?
+mimetype.config.initialAndOnError.cronExpression=0/10 * * * * ?
+
+# Optional property to specify an external file or directory that will be read for mimetype definitions from YAML
+# files (possibly added to a volume via k8 ConfigMaps).
+mimetype.config.dir=shared/classes/alfresco/extension/transform/mimetypes
+
+# Schedule for reading rendition config definitions dynamically. Initially checks every 10 seconds and then switches to
+# every hour after the configuration is read successfully. If there is a error later reading the config, the
+# checks return to every 10 seconds.
+rendition.config.cronExpression=2 30 0/1 * * ?
 rendition.config.initialAndOnError.cronExpression=0/10 * * * * ?
 
 # Optional property to specify an external file or directory that will be read for rendition definitions from YAML
 # files (possibly added to a volume via k8 ConfigMaps).
-rendition.config.dir=
+rendition.config.dir=shared/classes/alfresco/extension/transform/renditions
 
 # Optional property to specify an external file or directory that will be read for transformer json config.
-local.transform.pipeline.config.dir=
+local.transform.pipeline.config.dir=shared/classes/alfresco/extension/transform/pipelines
 
 # Used to disable transforms locally.
 local.transform.service.enabled=true
 
 # Schedule for reading local transform config, so that T-Engines and local pipeline config is dynamically
-# picked up, or reintegrated after an outage. Initially checks every 10 seconds and then switches to every 10 minutes
+# picked up, or reintegrated after an outage. Initially checks every 10 seconds and then switches to every hour
 # after the configuration is read successfully. If there is a error later reading the config, the checks return to
 # every 10 seconds.
-local.transform.service.cronExpression=0 0/10 * * * ?
+local.transform.service.cronExpression=4 30 0/1 * * ?
 local.transform.service.initialAndOnError.cronExpression=0/10 * * * * ?
 
 # Used to disable transforms that extend AbstractContentTransformer2