From 451bf0549c64c91ff531100809524b1aa5c57ddc Mon Sep 17 00:00:00 2001 From: Brian Long Date: Thu, 10 Dec 2020 15:28:57 -0500 Subject: [PATCH 1/4] v1.1-SNAPSHOT pom --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9f8dba4..d1bd42d 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ me.brianlong git-utils jar - 1.0-SNAPSHOT + 1.1-SNAPSHOT Git Utilities From 2c0503102fd03dad3940f0b118fd449d7f9efdd8 Mon Sep 17 00:00:00 2001 From: Brian Long Date: Wed, 6 Jan 2021 21:58:41 -0500 Subject: [PATCH 2/4] rebranded --- pom.xml | 2 +- .../java/{me/brianlong => com/inteligr8}/git/CachedGit.java | 2 +- .../brianlong => com/inteligr8}/git/CompositeIterator.java | 2 +- .../{me/brianlong => com/inteligr8}/git/CredentialedGit.java | 2 +- .../brianlong => com/inteligr8}/git/ExpiringMapListener.java | 2 +- .../java/{me/brianlong => com/inteligr8}/git/ExtendedGit.java | 2 +- .../brianlong => com/inteligr8}/git/LRUExpiringHashMap.java | 2 +- .../{me/brianlong => com/inteligr8}/git/ListeningMap.java | 2 +- .../brianlong => com/inteligr8}/git/LocalRepositoryCache.java | 2 +- .../java/{me/brianlong => com/inteligr8}/git/MapListener.java | 2 +- .../inteligr8}/git/UniquePriorityFifoQueue.java | 2 +- .../{me/brianlong => com/inteligr8}/git/CommandUnitTest.java | 2 +- .../inteligr8}/git/LocalRepositoryCacheUnitTest.java | 4 +++- src/test/resources/log4j2.xml | 2 +- 14 files changed, 16 insertions(+), 14 deletions(-) rename src/main/java/{me/brianlong => com/inteligr8}/git/CachedGit.java (96%) rename src/main/java/{me/brianlong => com/inteligr8}/git/CompositeIterator.java (97%) rename src/main/java/{me/brianlong => com/inteligr8}/git/CredentialedGit.java (98%) rename src/main/java/{me/brianlong => com/inteligr8}/git/ExpiringMapListener.java (83%) rename src/main/java/{me/brianlong => com/inteligr8}/git/ExtendedGit.java (99%) rename src/main/java/{me/brianlong => com/inteligr8}/git/LRUExpiringHashMap.java (99%) rename src/main/java/{me/brianlong => com/inteligr8}/git/ListeningMap.java (83%) rename src/main/java/{me/brianlong => com/inteligr8}/git/LocalRepositoryCache.java (99%) rename src/main/java/{me/brianlong => com/inteligr8}/git/MapListener.java (88%) rename src/main/java/{me/brianlong => com/inteligr8}/git/UniquePriorityFifoQueue.java (99%) rename src/test/java/{me/brianlong => com/inteligr8}/git/CommandUnitTest.java (98%) rename src/test/java/{me/brianlong => com/inteligr8}/git/LocalRepositoryCacheUnitTest.java (98%) diff --git a/pom.xml b/pom.xml index d1bd42d..ee3f903 100644 --- a/pom.xml +++ b/pom.xml @@ -2,7 +2,7 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - me.brianlong + com.inteligr8 git-utils jar 1.1-SNAPSHOT diff --git a/src/main/java/me/brianlong/git/CachedGit.java b/src/main/java/com/inteligr8/git/CachedGit.java similarity index 96% rename from src/main/java/me/brianlong/git/CachedGit.java rename to src/main/java/com/inteligr8/git/CachedGit.java index 0ad4815..01e50fb 100644 --- a/src/main/java/me/brianlong/git/CachedGit.java +++ b/src/main/java/com/inteligr8/git/CachedGit.java @@ -1,4 +1,4 @@ -package me.brianlong.git; +package com.inteligr8.git; import org.eclipse.jgit.api.CloneCommand; import org.eclipse.jgit.api.Git; diff --git a/src/main/java/me/brianlong/git/CompositeIterator.java b/src/main/java/com/inteligr8/git/CompositeIterator.java similarity index 97% rename from src/main/java/me/brianlong/git/CompositeIterator.java rename to src/main/java/com/inteligr8/git/CompositeIterator.java index 860994b..49beefe 100644 --- a/src/main/java/me/brianlong/git/CompositeIterator.java +++ b/src/main/java/com/inteligr8/git/CompositeIterator.java @@ -1,4 +1,4 @@ -package me.brianlong.git; +package com.inteligr8.git; import java.util.Collection; import java.util.Iterator; diff --git a/src/main/java/me/brianlong/git/CredentialedGit.java b/src/main/java/com/inteligr8/git/CredentialedGit.java similarity index 98% rename from src/main/java/me/brianlong/git/CredentialedGit.java rename to src/main/java/com/inteligr8/git/CredentialedGit.java index a5020a1..67c7ea6 100644 --- a/src/main/java/me/brianlong/git/CredentialedGit.java +++ b/src/main/java/com/inteligr8/git/CredentialedGit.java @@ -1,4 +1,4 @@ -package me.brianlong.git; +package com.inteligr8.git; import org.eclipse.jgit.api.CloneCommand; import org.eclipse.jgit.api.FetchCommand; diff --git a/src/main/java/me/brianlong/git/ExpiringMapListener.java b/src/main/java/com/inteligr8/git/ExpiringMapListener.java similarity index 83% rename from src/main/java/me/brianlong/git/ExpiringMapListener.java rename to src/main/java/com/inteligr8/git/ExpiringMapListener.java index 08802a4..dfee830 100644 --- a/src/main/java/me/brianlong/git/ExpiringMapListener.java +++ b/src/main/java/com/inteligr8/git/ExpiringMapListener.java @@ -1,4 +1,4 @@ -package me.brianlong.git; +package com.inteligr8.git; import java.util.Map.Entry; diff --git a/src/main/java/me/brianlong/git/ExtendedGit.java b/src/main/java/com/inteligr8/git/ExtendedGit.java similarity index 99% rename from src/main/java/me/brianlong/git/ExtendedGit.java rename to src/main/java/com/inteligr8/git/ExtendedGit.java index f9896ab..e4a3348 100644 --- a/src/main/java/me/brianlong/git/ExtendedGit.java +++ b/src/main/java/com/inteligr8/git/ExtendedGit.java @@ -1,4 +1,4 @@ -package me.brianlong.git; +package com.inteligr8.git; import java.io.IOException; import java.net.URISyntaxException; diff --git a/src/main/java/me/brianlong/git/LRUExpiringHashMap.java b/src/main/java/com/inteligr8/git/LRUExpiringHashMap.java similarity index 99% rename from src/main/java/me/brianlong/git/LRUExpiringHashMap.java rename to src/main/java/com/inteligr8/git/LRUExpiringHashMap.java index 08f4211..886941b 100644 --- a/src/main/java/me/brianlong/git/LRUExpiringHashMap.java +++ b/src/main/java/com/inteligr8/git/LRUExpiringHashMap.java @@ -1,4 +1,4 @@ -package me.brianlong.git; +package com.inteligr8.git; import java.io.Serializable; import java.util.Collection; diff --git a/src/main/java/me/brianlong/git/ListeningMap.java b/src/main/java/com/inteligr8/git/ListeningMap.java similarity index 83% rename from src/main/java/me/brianlong/git/ListeningMap.java rename to src/main/java/com/inteligr8/git/ListeningMap.java index f3cbf53..bd47e6d 100644 --- a/src/main/java/me/brianlong/git/ListeningMap.java +++ b/src/main/java/com/inteligr8/git/ListeningMap.java @@ -1,4 +1,4 @@ -package me.brianlong.git; +package com.inteligr8.git; import java.util.Map; diff --git a/src/main/java/me/brianlong/git/LocalRepositoryCache.java b/src/main/java/com/inteligr8/git/LocalRepositoryCache.java similarity index 99% rename from src/main/java/me/brianlong/git/LocalRepositoryCache.java rename to src/main/java/com/inteligr8/git/LocalRepositoryCache.java index b72db2c..47f1dfd 100644 --- a/src/main/java/me/brianlong/git/LocalRepositoryCache.java +++ b/src/main/java/com/inteligr8/git/LocalRepositoryCache.java @@ -1,4 +1,4 @@ -package me.brianlong.git; +package com.inteligr8.git; import java.io.File; import java.io.IOException; diff --git a/src/main/java/me/brianlong/git/MapListener.java b/src/main/java/com/inteligr8/git/MapListener.java similarity index 88% rename from src/main/java/me/brianlong/git/MapListener.java rename to src/main/java/com/inteligr8/git/MapListener.java index 47298f9..bf09a67 100644 --- a/src/main/java/me/brianlong/git/MapListener.java +++ b/src/main/java/com/inteligr8/git/MapListener.java @@ -1,4 +1,4 @@ -package me.brianlong.git; +package com.inteligr8.git; import java.util.Map.Entry; diff --git a/src/main/java/me/brianlong/git/UniquePriorityFifoQueue.java b/src/main/java/com/inteligr8/git/UniquePriorityFifoQueue.java similarity index 99% rename from src/main/java/me/brianlong/git/UniquePriorityFifoQueue.java rename to src/main/java/com/inteligr8/git/UniquePriorityFifoQueue.java index 5cb4981..c0aa0a0 100644 --- a/src/main/java/me/brianlong/git/UniquePriorityFifoQueue.java +++ b/src/main/java/com/inteligr8/git/UniquePriorityFifoQueue.java @@ -1,4 +1,4 @@ -package me.brianlong.git; +package com.inteligr8.git; import java.util.Arrays; import java.util.Collection; diff --git a/src/test/java/me/brianlong/git/CommandUnitTest.java b/src/test/java/com/inteligr8/git/CommandUnitTest.java similarity index 98% rename from src/test/java/me/brianlong/git/CommandUnitTest.java rename to src/test/java/com/inteligr8/git/CommandUnitTest.java index 3d05f81..565e69c 100644 --- a/src/test/java/me/brianlong/git/CommandUnitTest.java +++ b/src/test/java/com/inteligr8/git/CommandUnitTest.java @@ -1,4 +1,4 @@ -package me.brianlong.git; +package com.inteligr8.git; import java.io.File; import java.io.IOException; diff --git a/src/test/java/me/brianlong/git/LocalRepositoryCacheUnitTest.java b/src/test/java/com/inteligr8/git/LocalRepositoryCacheUnitTest.java similarity index 98% rename from src/test/java/me/brianlong/git/LocalRepositoryCacheUnitTest.java rename to src/test/java/com/inteligr8/git/LocalRepositoryCacheUnitTest.java index 6eb115e..3bc7600 100644 --- a/src/test/java/me/brianlong/git/LocalRepositoryCacheUnitTest.java +++ b/src/test/java/com/inteligr8/git/LocalRepositoryCacheUnitTest.java @@ -1,4 +1,4 @@ -package me.brianlong.git; +package com.inteligr8.git; import java.io.IOException; @@ -13,6 +13,8 @@ import org.junit.AfterClass; import org.junit.Assert; import org.junit.Test; +import com.inteligr8.git.LocalRepositoryCache; + public class LocalRepositoryCacheUnitTest { private final static CredentialsProvider gitCreds = new UsernamePasswordCredentialsProvider( diff --git a/src/test/resources/log4j2.xml b/src/test/resources/log4j2.xml index 7a59c70..60d0dce 100644 --- a/src/test/resources/log4j2.xml +++ b/src/test/resources/log4j2.xml @@ -6,7 +6,7 @@ - + From 909fde947777a66a23a0df12301f9c9634dffb42 Mon Sep 17 00:00:00 2001 From: Brian Long Date: Thu, 7 Jan 2021 11:18:31 -0500 Subject: [PATCH 3/4] refactored based on new LocalRepositoryCache unit tests --- .../java/com/inteligr8/git/CachedGit.java | 4 + .../com/inteligr8/git/DeveloperException.java | 15 ++ .../java/com/inteligr8/git/ExtendedGit.java | 4 - .../com/inteligr8/git/LRUExpiringHashMap.java | 138 ++++++++------- .../inteligr8/git/LocalRepositoryCache.java | 166 ++++++++++++------ .../git/LocalRepositoryCacheUnitTest.java | 115 +++++++++--- 6 files changed, 299 insertions(+), 143 deletions(-) create mode 100644 src/main/java/com/inteligr8/git/DeveloperException.java diff --git a/src/main/java/com/inteligr8/git/CachedGit.java b/src/main/java/com/inteligr8/git/CachedGit.java index 01e50fb..001b346 100644 --- a/src/main/java/com/inteligr8/git/CachedGit.java +++ b/src/main/java/com/inteligr8/git/CachedGit.java @@ -21,6 +21,10 @@ public class CachedGit extends Git { super(repo); } + public String getFirstRemoteUrl() throws GitAPIException { + return this.remoteList().call().iterator().next().getURIs().iterator().next().toString(); + } + @Override public void close() { LocalRepositoryCache.getInstance().release(this); diff --git a/src/main/java/com/inteligr8/git/DeveloperException.java b/src/main/java/com/inteligr8/git/DeveloperException.java new file mode 100644 index 0000000..906c20d --- /dev/null +++ b/src/main/java/com/inteligr8/git/DeveloperException.java @@ -0,0 +1,15 @@ +package com.inteligr8.git; + +import org.slf4j.Logger; + +public class DeveloperException extends RuntimeException { + + private static final long serialVersionUID = 1600616227925437703L; + private static final String MESSAGE = "This is a bug that needs to be addressed by the developer"; + + public DeveloperException(Logger logger, Throwable cause) { + super(MESSAGE, cause); + logger.error(MESSAGE); + } + +} diff --git a/src/main/java/com/inteligr8/git/ExtendedGit.java b/src/main/java/com/inteligr8/git/ExtendedGit.java index e4a3348..427f5fb 100644 --- a/src/main/java/com/inteligr8/git/ExtendedGit.java +++ b/src/main/java/com/inteligr8/git/ExtendedGit.java @@ -47,10 +47,6 @@ public class ExtendedGit extends CachedGit { super(repo); } - public String getFirstRemoteUrl() throws GitAPIException { - return this.remoteList().call().iterator().next().getURIs().iterator().next().toString(); - } - public String getRepositoryFullyQualifiedName() throws GitAPIException, URISyntaxException { String gitUrl = this.getFirstRemoteUrl(); if (this.logger.isDebugEnabled()) diff --git a/src/main/java/com/inteligr8/git/LRUExpiringHashMap.java b/src/main/java/com/inteligr8/git/LRUExpiringHashMap.java index 886941b..d98b7b8 100644 --- a/src/main/java/com/inteligr8/git/LRUExpiringHashMap.java +++ b/src/main/java/com/inteligr8/git/LRUExpiringHashMap.java @@ -17,24 +17,25 @@ public class LRUExpiringHashMap implements ListeningM private final Logger logger = LoggerFactory.getLogger(LRUExpiringHashMap.class); - private final Map map; + private final Object sync = new Object(); + private final LinkedHashMap map; private final List> listeners = new LinkedList>(); private final long expirationTimeMillis; public LRUExpiringHashMap(int expirationTimeInMinutes) { - this.map = new LinkedHashMap(); + this.map = new LinkedHashMap<>(); this.expirationTimeMillis = expirationTimeInMinutes * 60L + 1000L; } public LRUExpiringHashMap(int expirationTimeInMinutes, int initialCapacity) { - this.map = new LinkedHashMap(initialCapacity); + this.map = new LinkedHashMap<>(initialCapacity); this.expirationTimeMillis = expirationTimeInMinutes * 60L + 1000L; } @Override public void addListener(MapListener listener) { if (this.logger.isDebugEnabled()) - this.logger.debug("adding listener"); + this.logger.debug("adding listener: " + listener.getClass()); this.listeners.add(listener); } @@ -76,47 +77,60 @@ public class LRUExpiringHashMap implements ListeningM @Override @SuppressWarnings("unchecked") public V get(Object key) { - if (!this.map.containsKey(key)) - return null; + ExpiringValue evalue; - // remove and put to move the entry to the end of the map; helping with finding expired entries - V value = this.map.remove(key); - this.map.put(new ExpiringHashKey((K)key, System.currentTimeMillis() + this.expirationTimeMillis), value); + synchronized (this.sync) { + if (!this.map.containsKey(key)) + return null; + + // remove and put to move the entry to the end of the map; helping with finding expired entries + evalue = this.map.remove(key); + this.map.put((K)key, new ExpiringValue(evalue.getValue(), System.currentTimeMillis() + this.expirationTimeMillis)); + } for (MapListener listener : this.listeners) - listener.accessed(new ExpiringHashMapEntry((K)key, value)); - return value; + listener.accessed(new ExpiringHashMapEntry((K)key, evalue.getValue())); + return evalue.getValue(); } @Override public V put(K key, V value) { - ExpiringHashKey ehkey = new ExpiringHashKey(key, System.currentTimeMillis() + this.expirationTimeMillis); - V oldValue = this.map.put(ehkey, value); + ExpiringValue evalue = new ExpiringValue(value, System.currentTimeMillis() + this.expirationTimeMillis); + ExpiringValue oldValue = this.map.put(key, evalue); for (MapListener listener : this.listeners) listener.added(new ExpiringHashMapEntry(key, value)); - return oldValue; + return oldValue == null ? null : oldValue.getValue(); } @Override public void clear() { - for (Entry entry : this.map.entrySet()) { - for (MapListener listener : this.listeners) - listener.cleared(new ExpiringHashMapEntry(entry.getKey().getKey(), entry.getValue())); - } + List entries = new LinkedList<>(); - this.map.clear(); + synchronized (this.sync) { + for (Entry entry : this.map.entrySet()) + entries.add(new ExpiringHashMapEntry(entry.getKey(), entry.getValue().getValue())); + this.map.clear(); + } + + for (ExpiringHashMapEntry entry : entries) + for (MapListener listener : this.listeners) + listener.cleared(entry); } @SuppressWarnings("unchecked") @Override public V remove(Object key) { - if (!this.map.containsKey(key)) - return null; + ExpiringValue evalue; + + synchronized (this.sync) { + if (!this.map.containsKey(key)) + return null; + evalue = this.map.remove(key); + } - V value = this.map.remove(key); for (MapListener listener : this.listeners) - listener.removed(new ExpiringHashMapEntry((K)key, value)); - return value; + listener.removed(new ExpiringHashMapEntry((K)key, evalue.getValue())); + return evalue.getValue(); } @Override @@ -135,28 +149,49 @@ public class LRUExpiringHashMap implements ListeningM } public void exipriationCheck() { - Iterator> i = this.map.entrySet().iterator(); - for (Entry entry = i.next(); i.hasNext(); entry = i.next()) { - if (entry.getKey().isExpired()) { - i.remove(); - for (MapListener listener : this.listeners) - if (listener instanceof ExpiringMapListener) - ((ExpiringMapListener)listener).expired(new ExpiringHashMapEntry(entry.getKey().getKey(), entry.getValue())); + // since these should be order, we could break out of this loop once we reach an unexpired entry + List entries = new LinkedList<>(); + + synchronized (this.sync) { + Iterator> i = this.map.entrySet().iterator(); + for (Entry entry = i.next(); i.hasNext(); entry = i.next()) { + if (entry.getValue().isExpired()) { + i.remove(); + entries.add(new ExpiringHashMapEntry(entry.getKey(), entry.getValue().getValue())); + } else { + // ordered map; ordered by time of entry + // since expiration timers are constant, we can skip looping + break; + } } } + + for (ExpiringHashMapEntry entry : entries) + for (MapListener listener : this.listeners) + if (listener instanceof ExpiringMapListener) + ((ExpiringMapListener)listener).expired(entry); } public void expire(K key) { if (this.logger.isDebugEnabled()) this.logger.debug("expiring key from map: " + key); - - if (!this.map.containsKey(key)) - return; + + ExpiringValue evalue; - V value = this.map.remove(key); + synchronized (this.sync) { + if (!this.map.containsKey(key)) + return; + evalue = this.map.remove(key); + } + for (MapListener listener : this.listeners) if (listener instanceof ExpiringMapListener) - ((ExpiringMapListener)listener).expired(new ExpiringHashMapEntry(key, value)); + ((ExpiringMapListener)listener).expired(new ExpiringHashMapEntry(key, evalue.getValue())); + } + + @Override + public String toString() { + return this.map.toString(); } @@ -207,44 +242,27 @@ public class LRUExpiringHashMap implements ListeningM } - private class ExpiringHashKey implements Serializable { - - private static final long serialVersionUID = -6511298315143655313L; + private class ExpiringValue { private long expirationTimeInMillis; - private K key; + private V value; - public ExpiringHashKey(K key, long expirationTimeInMillis) { - this.key = key; + public ExpiringValue(V value, long expirationTimeInMillis) { + this.value = value; this.expirationTimeInMillis = expirationTimeInMillis; } - public K getKey() { - return this.key; + public V getValue() { + return this.value; } public boolean isExpired() { return this.expirationTimeInMillis <= System.currentTimeMillis(); } - @Override - public int hashCode() { - return this.key.hashCode(); - } - @Override public String toString() { - return this.key.toString(); - } - - @Override - @SuppressWarnings("unchecked") - public boolean equals(Object obj) { - if (obj instanceof LRUExpiringHashMap.ExpiringHashKey) { - return this.key.equals(((ExpiringHashKey)obj).key); - } else { - return this.key.equals(obj); - } + return this.value.toString(); } } diff --git a/src/main/java/com/inteligr8/git/LocalRepositoryCache.java b/src/main/java/com/inteligr8/git/LocalRepositoryCache.java index 47f1dfd..1b1c07d 100644 --- a/src/main/java/com/inteligr8/git/LocalRepositoryCache.java +++ b/src/main/java/com/inteligr8/git/LocalRepositoryCache.java @@ -2,14 +2,20 @@ package com.inteligr8.git; import java.io.File; import java.io.IOException; +import java.net.URISyntaxException; +import java.util.HashMap; +import java.util.Map; import java.util.Map.Entry; import java.util.UUID; +import java.util.concurrent.Semaphore; import org.eclipse.jgit.api.CloneCommand; import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.ResetCommand.ResetType; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.api.errors.InvalidRemoteException; import org.eclipse.jgit.api.errors.RefNotFoundException; +import org.eclipse.jgit.api.errors.TransportException; import org.eclipse.jgit.transport.CredentialsProvider; import org.eclipse.jgit.util.FileUtils; import org.slf4j.Logger; @@ -23,15 +29,18 @@ public class LocalRepositoryCache { return INSTANCE; } + + private final Logger logger = LoggerFactory.getLogger(LocalRepositoryCache.class); - private final LRUExpiringHashMap cachedGits = new LRUExpiringHashMap(30); -// private final Map gitIdsToUrls = new HashMap(); - private final File cacheDirectory = new File(System.getProperty("java.io.tmpdir"), "git"); + private final LRUExpiringHashMap cachedGits = new LRUExpiringHashMap<>(30); + private final Map gitUrlSemaphores = new HashMap<>(); + final File cacheDirectory = new File(System.getProperty("java.io.tmpdir"), "git"); + private final int simultaneousProcessesPerGitRepo = 1; private LocalRepositoryCache() { this.cacheDirectory.mkdir(); - this.cachedGits.addListener(new RepositoryCacheMapListener()); + this.cachedGits.addListener(new RepositoryCacheMapListener()); } @Override @@ -49,74 +58,123 @@ public class LocalRepositoryCache { this.clear(); } - public synchronized ExtendedGit acquire(String url) throws GitAPIException, InvalidRemoteException, RefNotFoundException { + public CachedGit acquire(String url) throws InterruptedException, InvalidRemoteException, RefNotFoundException, TransportException, GitAPIException { return this.acquire(url, null, null); } - public synchronized ExtendedGit acquire(String url, CredentialsProvider creds) throws GitAPIException, InvalidRemoteException, RefNotFoundException { + public CachedGit acquire(String url, CredentialsProvider creds) throws InterruptedException, InvalidRemoteException, RefNotFoundException, TransportException, GitAPIException { return this.acquire(url, creds, null); } - public synchronized ExtendedGit acquire(String url, String branch) throws GitAPIException, InvalidRemoteException, RefNotFoundException { + public CachedGit acquire(String url, String branch) throws InterruptedException, InvalidRemoteException, RefNotFoundException, TransportException, GitAPIException { return this.acquire(url, null, branch); } - public synchronized ExtendedGit acquire(String url, CredentialsProvider creds, String branch) throws GitAPIException, InvalidRemoteException, RefNotFoundException { + /** + * + * @param url A URL for the Git Repository. + * @param creds A username/password for the Git Repository. + * @param branch A branch name of the branch in the Git Repository to start working on. + * @return A JGit object. + * @throws InterruptedException The JVM was interrupted while the thread was waiting for another request on the same Git Repository + * @throws InvalidRemoteException The URL refers to nothing or an invalid Git Repository + * @throws RefNotFoundException The branch does not exist in the Git Repository represented by the URL + * @throws TransportException A disk or network error or hiccup occurred. + * @throws GitAPIException A Git API, library, or protocol related issue occurred + */ + public CachedGit acquire(String url, CredentialsProvider creds, String branch) + throws InterruptedException, InvalidRemoteException, RefNotFoundException, TransportException, GitAPIException { if (this.logger.isTraceEnabled()) this.logger.trace("acquire('" + url + "', " + creds + ", '" + branch + "')"); -// Git git = this.cachedGits.remove(url); -// if (git == null) { - if (this.logger.isDebugEnabled()) - this.logger.debug("creating temporary Git directory"); - File gitRepoDirectory = new File(this.cacheDirectory, UUID.randomUUID().toString() + ".git"); - - CloneCommand clone = new CloneCommand() - .setURI(url) - .setDirectory(gitRepoDirectory); - if (branch != null) - clone.setBranch(branch); - - if (this.logger.isDebugEnabled()) - this.logger.debug("cloning Git repository: " + url); - ExtendedGit git = creds != null ? new CredentialedGit(clone, creds) : new ExtendedGit(clone); - if (this.logger.isInfoEnabled()) - this.logger.info("Cloned Git Repository"); + Semaphore semaphore = null; + synchronized (this) { + semaphore = this.gitUrlSemaphores.get(url); + if (semaphore == null) + this.gitUrlSemaphores.put(url, semaphore = new Semaphore(this.simultaneousProcessesPerGitRepo)); + } + + semaphore.acquire(); + try { + CachedGit git = this.cachedGits.remove(url); + if (git == null) { + File gitRepoDirectory = new File(this.cacheDirectory, UUID.randomUUID().toString() + ".git"); + if (this.logger.isDebugEnabled()) + this.logger.debug("Git directory cache for clone: " + gitRepoDirectory); + + CloneCommand clone = new CloneCommand() + .setURI(url) + .setDirectory(gitRepoDirectory); + if (branch != null) + clone.setBranch(branch); + + if (this.logger.isDebugEnabled()) + this.logger.debug("Cloning Git repository: " + url); + git = creds != null ? new CredentialedGit(clone, creds) : new ExtendedGit(clone); + if (this.logger.isInfoEnabled()) + this.logger.info("Cloned Git Repository: " + ((ExtendedGit)git).getRepositoryFullyQualifiedName()); + } else { + if (this.logger.isDebugEnabled()) + this.logger.debug("reseting Git"); + git.reset().setMode(ResetType.HARD).call(); + + if (branch != null) { + if (this.logger.isDebugEnabled()) + this.logger.debug("switching Git branches: " + branch); + git.checkout().setName(branch).call(); + } + + if (this.logger.isDebugEnabled()) + this.logger.debug("updating Git branch: " + git.getRepository().getBranch()); + git.pull().call(); + } + return git; -// git = creds != null ? new CredentialedGit(clone, creds) : new CachedGit(clone); -// this.gitIdsToUrls.put(git.getRepository().getIdentifier(), url); -// } else { -// if (branch != null) { -// if (this.logger.isDebugEnabled()) -// this.logger.debug("switching Git branches: " + branch); -// git.checkout().setName(branch).call(); -// } -// -// if (this.logger.isDebugEnabled()) -// this.logger.debug("updating Git branch: " + branch); -// git.pull().call(); -// } + } catch (URISyntaxException use) { + semaphore.release(); + throw new DeveloperException(this.logger, use); + } catch (IOException ie) { + semaphore.release(); + throw new TransportException("A I/O issue occurred", ie); + } catch (GitAPIException gae) { + semaphore.release(); + throw gae; + } } - public synchronized void release(Git git) { + public void release(CachedGit git) { if (this.logger.isTraceEnabled()) this.logger.trace("release('" + git.getRepository().getIdentifier() + "')"); -// String url = this.gitIdsToUrls.get(git.getRepository().getIdentifier()); -// this.cachedGits.put(url, git); - this.expunge(git); + try { + String url = git.getFirstRemoteUrl(); + + synchronized (this) { + Semaphore semaphore = this.gitUrlSemaphores.get(url); + if (semaphore != null) + semaphore.release(); + } + + this.cachedGits.put(url, git); + } catch (GitAPIException gae) { + if (this.logger.isDebugEnabled()) + this.logger.debug("A Git repository was never released", gae); + this.logger.warn("A Git repository was never released, potentially blocking all future processing by this tool until a restart: " + git.toString()); + } } - public synchronized void clear() { + public void clear() { if (this.logger.isTraceEnabled()) this.logger.trace("clear()"); -// this.cachedGits.clear(); + this.cachedGits.clear(); + + for (Semaphore semaphore : this.gitUrlSemaphores.values()) + ; // FIXME interrupt all the threads; releasing semaphores will not be enough + this.gitUrlSemaphores.clear(); } private void expunge(Git git) { -// gitIdsToUrls.remove(git.getRepository().getIdentifier()); - File gitDir = git.getRepository().getDirectory(); File workingTreeDir = git.getRepository().getWorkTree(); git.getRepository().close(); @@ -129,7 +187,6 @@ public class LocalRepositoryCache { this.logger.warn("Failed to delete a git directory: " + gitDir); if (this.logger.isDebugEnabled()) this.logger.debug(ie.getMessage(), ie); - gitDir.deleteOnExit(); } try { @@ -140,42 +197,41 @@ public class LocalRepositoryCache { this.logger.warn("Failed to delete a git directory: " + workingTreeDir); if (this.logger.isDebugEnabled()) this.logger.debug(ie.getMessage(), ie); - workingTreeDir.deleteOnExit(); } if (this.logger.isInfoEnabled()) - this.logger.info("Deleted Git Repository"); + this.logger.info("Deleted Git Repository: " + gitDir); } - private class RepositoryCacheMapListener implements ExpiringMapListener { + private class RepositoryCacheMapListener implements ExpiringMapListener { private final Logger logger = LoggerFactory.getLogger(LocalRepositoryCache.class); @Override - public void accessed(Entry entry) { + public void accessed(Entry entry) { } @Override - public void added(Entry entry) { + public void added(Entry entry) { // a clean one or one returned after being previously removed } @Override - public void expired(Entry entry) { + public void expired(Entry entry ) { if (this.logger.isTraceEnabled()) this.logger.trace("expired('" + entry.getKey() + "', '" + entry.getValue().getRepository().getIdentifier() + "')"); expunge(entry.getValue()); } @Override - public void removed(Entry entry) { + public void removed(Entry entry) { // expected to be removed only temporarily...for use elsewhere; do not close } @Override - public void cleared(Entry entry) { + public void cleared(Entry entry) { if (this.logger.isTraceEnabled()) this.logger.trace("cleared('" + entry.getKey() + "', '" + entry.getValue().getRepository().getIdentifier() + "')"); expunge(entry.getValue()); diff --git a/src/test/java/com/inteligr8/git/LocalRepositoryCacheUnitTest.java b/src/test/java/com/inteligr8/git/LocalRepositoryCacheUnitTest.java index 3bc7600..ec398b2 100644 --- a/src/test/java/com/inteligr8/git/LocalRepositoryCacheUnitTest.java +++ b/src/test/java/com/inteligr8/git/LocalRepositoryCacheUnitTest.java @@ -9,20 +9,27 @@ import org.eclipse.jgit.api.errors.TransportException; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.transport.CredentialsProvider; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; -import org.junit.AfterClass; +import org.junit.After; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; -import com.inteligr8.git.LocalRepositoryCache; - public class LocalRepositoryCacheUnitTest { - + + private final static CredentialsProvider badCreds = new UsernamePasswordCredentialsProvider("not-a-valid-user", "not-a-valid-password"); private final static CredentialsProvider gitCreds = new UsernamePasswordCredentialsProvider( - System.getProperty("git.username"), System.getProperty("git.token")); + System.getProperty("bitbucket.inteligr8.username"), System.getProperty("bitbucket.inteligr8.token")); - @AfterClass - public static void cleanup() { + private int cachedFiles; + + @Before + public void compileStats() { + this.cachedFiles = LocalRepositoryCache.getInstance().cacheDirectory.listFiles().length; + } + @After + public void cleanupAndValidate() { LocalRepositoryCache.getInstance().clear(); + Assert.assertEquals(this.cachedFiles, LocalRepositoryCache.getInstance().cacheDirectory.listFiles().length); } /** @@ -30,7 +37,7 @@ public class LocalRepositoryCacheUnitTest { * is thrown. It is wrapped inside a JGit TransportException. */ @Test(expected = TransportException.class) - public void cacheBadHostRepo() throws GitAPIException { + public void tryBadHostRepo() throws GitAPIException, InterruptedException { LocalRepositoryCache.getInstance().acquire("https://host-does-not-exist.com/bmlong137/does-not-exist.git"); } @@ -42,7 +49,7 @@ public class LocalRepositoryCacheUnitTest { * more like cacheNonExistentRepoAuth() than cacheNonExistentRepo(). */ @Test(expected = InvalidRemoteException.class) - public void cacheNonExistentRepoViaSsh() throws GitAPIException { + public void tryNonExistentRepoViaSsh() throws GitAPIException, InterruptedException { LocalRepositoryCache.getInstance().acquire("git@github.com:bmlong137/does-not-exist.git"); } @@ -52,7 +59,7 @@ public class LocalRepositoryCacheUnitTest { * which is wrapped inside a JGit TransportException. */ @Test(expected = TransportException.class) - public void cacheNonExistentRepo() throws GitAPIException { + public void tryNonExistentRepo() throws GitAPIException, InterruptedException { LocalRepositoryCache.getInstance().acquire("https://github.com/bmlong137/does-not-exist.git"); } @@ -61,13 +68,42 @@ public class LocalRepositoryCacheUnitTest { * thrown. It is wrapped inside a JGit InvalidRemoteException. */ @Test(expected = InvalidRemoteException.class) - public void cacheNonExistentRepoAuth() throws GitAPIException { - LocalRepositoryCache.getInstance().acquire("https://github.com/bmlong137/does-not-exist.git", gitCreds); + public void tryNonExistentRepoAuth() throws GitAPIException, InterruptedException { + LocalRepositoryCache.getInstance().acquire("https://bitbucket.org/inteligr8/does-not-exist.git", gitCreds); + } + + /** + * Invalid credentials and repo doesn't actually exist + */ + @Test(expected = InvalidRemoteException.class) + public void tryInvalidCredsOnNoRepo() throws GitAPIException, InterruptedException { + LocalRepositoryCache.getInstance().acquire("https://bitbucket.org/inteligr8/does-not-exist.git", badCreds); + } + + /** + * Invalid credentials and public repo exists + */ + @Test + public void cacheInvalidCredsOnPublicRepo() throws GitAPIException, InterruptedException { + Git git = LocalRepositoryCache.getInstance().acquire("https://bitbucket.org/inteligr8/git-utils.git", badCreds); + try { + this.validateGenericGitRepo(git); + } finally { + git.close(); + } + } + + /** + * Invalid credentials and private repo exists + */ + @Test(expected = TransportException.class) + public void tryInvalidCredsOnPrivateRepo() throws GitAPIException, InterruptedException { + LocalRepositoryCache.getInstance().acquire("https://bitbucket.org/inteligr8/propagate-branches.git", badCreds); } @Test - public void cachePublicRepo() throws GitAPIException { - Git git = LocalRepositoryCache.getInstance().acquire("https://github.com/bmlong137/maven-file-management.git"); + public void cachePublicRepo() throws GitAPIException, InterruptedException { + Git git = LocalRepositoryCache.getInstance().acquire("https://bitbucket.org/inteligr8/git-utils.git"); try { this.validateGenericGitRepo(git); } finally { @@ -76,8 +112,8 @@ public class LocalRepositoryCacheUnitTest { } @Test - public void cachePublicRepoViaSsh() throws GitAPIException { - Git git = LocalRepositoryCache.getInstance().acquire("git@github.com:bmlong137/maven-file-management.git"); + public void cachePublicRepoViaSsh() throws GitAPIException, InterruptedException { + Git git = LocalRepositoryCache.getInstance().acquire("git@bitbucket.org:inteligr8/git-utils.git"); try { this.validateGenericGitRepo(git); } finally { @@ -86,18 +122,18 @@ public class LocalRepositoryCacheUnitTest { } /** - * Since "github-api" isn't a public repo, it requires authentication + * Since "propagate-branches" isn't a public repo, it requires authentication * to even check to see if it is private. This causes a JSchException * which is wrapped inside a JGit TransportException. */ @Test(expected = TransportException.class) - public void cachePrivateRepoUnauth() throws GitAPIException { - LocalRepositoryCache.getInstance().acquire("https://github.com/bmlong137/github-api.git"); + public void tryPrivateRepoUnauth() throws GitAPIException, InterruptedException { + LocalRepositoryCache.getInstance().acquire("https://bitbucket.org/inteligr8/propagate-branches.git"); } @Test - public void cachePrivateRepo() throws GitAPIException { - Git git = LocalRepositoryCache.getInstance().acquire("https://github.com/bmlong137/github-api.git", gitCreds); + public void cachePrivateRepo() throws GitAPIException, InterruptedException { + Git git = LocalRepositoryCache.getInstance().acquire("https://bitbucket.org/inteligr8/propagate-branches.git", gitCreds); try { this.validateGenericGitRepo(git); } finally { @@ -106,12 +142,43 @@ public class LocalRepositoryCacheUnitTest { } @Test - public void cachePublicRepoBranch() throws GitAPIException, IOException { - Git git = LocalRepositoryCache.getInstance().acquire("https://github.com/bmlong137/maven-file-management.git", "master"); + public void cachePublicRepoBranch() throws GitAPIException, InterruptedException, IOException { + Git git = LocalRepositoryCache.getInstance().acquire("https://bitbucket.org/inteligr8/git-utils.git", "stable"); try { this.validateGenericGitRepo(git); - Assert.assertEquals("master", git.getRepository().getBranch()); + Assert.assertEquals("stable", git.getRepository().getBranch()); + } finally { + git.close(); + } + } + + @Test + public void cacheReuse() throws GitAPIException, InterruptedException, IOException { + Git git = LocalRepositoryCache.getInstance().acquire("https://bitbucket.org/inteligr8/git-utils.git"); + String repoId = git.getRepository().getIdentifier(); + git.close(); + + Assert.assertEquals(this.cachedFiles+1, LocalRepositoryCache.getInstance().cacheDirectory.listFiles().length); + Thread.sleep(1000L); + Assert.assertEquals(this.cachedFiles+1, LocalRepositoryCache.getInstance().cacheDirectory.listFiles().length); + + git = LocalRepositoryCache.getInstance().acquire("https://bitbucket.org/inteligr8/git-utils.git"); + try { + Assert.assertEquals(repoId, git.getRepository().getIdentifier()); + git.checkout().setName("develop").setStartPoint("origin/develop").setCreateBranch(true).call(); + Assert.assertEquals("develop", git.getRepository().getBranch()); + } finally { + git.close(); + } + + Assert.assertEquals(this.cachedFiles+1, LocalRepositoryCache.getInstance().cacheDirectory.listFiles().length); + Thread.sleep(1000L); + Assert.assertEquals(this.cachedFiles+1, LocalRepositoryCache.getInstance().cacheDirectory.listFiles().length); + + git = LocalRepositoryCache.getInstance().acquire("https://bitbucket.org/inteligr8/git-utils.git", "stable"); + try { + Assert.assertEquals("stable", git.getRepository().getBranch()); } finally { git.close(); } From 06db1d5d4590a3709dd96a441e6f1b00567d10e5 Mon Sep 17 00:00:00 2001 From: Brian Long Date: Thu, 7 Jan 2021 11:24:11 -0500 Subject: [PATCH 4/4] updated command test; ready for more tests in the future --- .../com/inteligr8/git/CommandUnitTest.java | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/test/java/com/inteligr8/git/CommandUnitTest.java b/src/test/java/com/inteligr8/git/CommandUnitTest.java index 565e69c..84be925 100644 --- a/src/test/java/com/inteligr8/git/CommandUnitTest.java +++ b/src/test/java/com/inteligr8/git/CommandUnitTest.java @@ -2,6 +2,7 @@ package com.inteligr8.git; import java.io.File; import java.io.IOException; +import java.util.Collections; import java.util.List; import java.util.UUID; @@ -10,8 +11,6 @@ import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.ListBranchCommand.ListMode; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.transport.CredentialsProvider; -import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; import org.eclipse.jgit.util.FileUtils; import org.junit.AfterClass; import org.junit.Assert; @@ -19,9 +18,6 @@ import org.junit.BeforeClass; import org.junit.Test; public class CommandUnitTest { - - private final static CredentialsProvider gitCreds = new UsernamePasswordCredentialsProvider( - System.getProperty("git.username"), System.getProperty("git.token")); private static File tmpdir; private static Git git; @@ -31,8 +27,7 @@ public class CommandUnitTest { tmpdir = new File(System.getProperty("java.io.tmpdir"), "git-" + UUID.randomUUID().toString() + ".tmp"); git = new CloneCommand() - .setURI("git@github.com:bmlong137/env-docker-adbp.git") - .setCredentialsProvider(gitCreds) + .setURI("git@bitbucket.org:inteligr8/git-utils.git") .setDirectory(tmpdir) .call(); } @@ -45,10 +40,17 @@ public class CommandUnitTest { } @Test - public void lotsOfBranches() throws GitAPIException { + public void listOfBranches() throws GitAPIException, IOException { List remoteBranches = git.branchList().setListMode(ListMode.REMOTE).call(); Assert.assertNotNull(remoteBranches); - Assert.assertTrue(remoteBranches.size() > 5); + Assert.assertTrue(remoteBranches.contains(git.getRepository().findRef("origin/develop"))); + Assert.assertTrue(remoteBranches.contains(git.getRepository().findRef("origin/stable"))); + + List localBranches = git.branchList().call(); + Assert.assertNotNull(localBranches); + Assert.assertEquals(1, localBranches.size()); + + Assert.assertTrue(Collections.disjoint(remoteBranches, localBranches)); } }