Commit 909fde94 authored by Brian Long's avatar Brian Long
Browse files

refactored based on new LocalRepositoryCache unit tests

parent 2c050310

src/main/java/com/inteligr8/git/CachedGit.java

+4 −0
Original line number Diff line number Diff line
@@ -21,6 +21,10 @@ public class CachedGit extends Git { 
		super(repo);

	}

	

	public String getFirstRemoteUrl() throws GitAPIException {

		return this.remoteList().call().iterator().next().getURIs().iterator().next().toString();

	}

	

	@Override

	public void close() {

		LocalRepositoryCache.getInstance().release(this);

src/main/java/com/inteligr8/git/DeveloperException.java

0 → 100644
+15 −0
Original line number Diff line number Diff line 
package com.inteligr8.git;



import org.slf4j.Logger;



public class DeveloperException extends RuntimeException {

	

	private static final long serialVersionUID = 1600616227925437703L;

	private static final String MESSAGE = "This is a bug that needs to be addressed by the developer";

	

	public DeveloperException(Logger logger, Throwable cause) {

		super(MESSAGE, cause);

		logger.error(MESSAGE);

	}



}

src/main/java/com/inteligr8/git/ExtendedGit.java

+0 −4
Original line number Diff line number Diff line
@@ -47,10 +47,6 @@ public class ExtendedGit extends CachedGit { 
		super(repo);

	}

	

	public String getFirstRemoteUrl() throws GitAPIException {

		return this.remoteList().call().iterator().next().getURIs().iterator().next().toString();

	}

	

	public String getRepositoryFullyQualifiedName() throws GitAPIException, URISyntaxException {

		String gitUrl = this.getFirstRemoteUrl();

		if (this.logger.isDebugEnabled())

src/main/java/com/inteligr8/git/LRUExpiringHashMap.java

+78 −60
Original line number Diff line number Diff line
@@ -17,24 +17,25 @@ public class LRUExpiringHashMap<K extends Serializable, V> implements ListeningM 


	private final Logger logger = LoggerFactory.getLogger(LRUExpiringHashMap.class);

	

	private final Map<ExpiringHashKey, V> map;

	private final Object sync = new Object();

	private final LinkedHashMap<K, ExpiringValue> map;

	private final List<MapListener<K, V>> listeners = new LinkedList<MapListener<K, V>>();

	private final long expirationTimeMillis;

	

	public LRUExpiringHashMap(int expirationTimeInMinutes) {

		this.map = new LinkedHashMap<ExpiringHashKey, V>();

		this.map = new LinkedHashMap<>();

		this.expirationTimeMillis = expirationTimeInMinutes * 60L + 1000L;

	}



	public LRUExpiringHashMap(int expirationTimeInMinutes, int initialCapacity) {

		this.map = new LinkedHashMap<ExpiringHashKey, V>(initialCapacity);

		this.map = new LinkedHashMap<>(initialCapacity);

		this.expirationTimeMillis = expirationTimeInMinutes * 60L + 1000L;

	}

	

	@Override

	public void addListener(MapListener<K, V> listener) {

    	if (this.logger.isDebugEnabled())

    		this.logger.debug("adding listener");

    		this.logger.debug("adding listener: " + listener.getClass());

		this.listeners.add(listener);

	}
@@ -76,47 +77,60 @@ public class LRUExpiringHashMap<K extends Serializable, V> implements ListeningM 
	@Override

	@SuppressWarnings("unchecked")

	public V get(Object key) {

		ExpiringValue evalue;

		

		synchronized (this.sync) {

			if (!this.map.containsKey(key))

				return null;

			

			// remove and put to move the entry to the end of the map; helping with finding expired entries

		V value = this.map.remove(key);

		this.map.put(new ExpiringHashKey((K)key, System.currentTimeMillis() + this.expirationTimeMillis), value);

			evalue = this.map.remove(key);

			this.map.put((K)key, new ExpiringValue(evalue.getValue(), System.currentTimeMillis() + this.expirationTimeMillis));

		}

		

		for (MapListener<K, V> listener : this.listeners)

			listener.accessed(new ExpiringHashMapEntry((K)key, value));

		return value;

			listener.accessed(new ExpiringHashMapEntry((K)key, evalue.getValue()));

		return evalue.getValue();

	}

	

	@Override

	public V put(K key, V value) {

		ExpiringHashKey ehkey = new ExpiringHashKey(key, System.currentTimeMillis() + this.expirationTimeMillis);

		V oldValue = this.map.put(ehkey, value);

		ExpiringValue evalue = new ExpiringValue(value, System.currentTimeMillis() + this.expirationTimeMillis);

		ExpiringValue oldValue = this.map.put(key, evalue);

		for (MapListener<K, V> listener : this.listeners)

			listener.added(new ExpiringHashMapEntry(key, value));

		return oldValue;

		return oldValue == null ? null : oldValue.getValue();

	}

	

	@Override

	public void clear() {

		for (Entry<ExpiringHashKey, V> entry : this.map.entrySet()) {

			for (MapListener<K, V> listener : this.listeners)

				listener.cleared(new ExpiringHashMapEntry(entry.getKey().getKey(), entry.getValue()));

		}

		List<ExpiringHashMapEntry> entries = new LinkedList<>();

		

		synchronized (this.sync) {

			for (Entry<K, ExpiringValue> entry : this.map.entrySet())

				entries.add(new ExpiringHashMapEntry(entry.getKey(), entry.getValue().getValue()));

			this.map.clear();

		}



		for (ExpiringHashMapEntry entry : entries)

			for (MapListener<K, V> listener : this.listeners)

				listener.cleared(entry);

	}

	

	@SuppressWarnings("unchecked")

	@Override

	public V remove(Object key) {

		ExpiringValue evalue;

		

		synchronized (this.sync) {

			if (!this.map.containsKey(key))

				return null;

			evalue = this.map.remove(key);

		}

		

		V value = this.map.remove(key);

		for (MapListener<K, V> listener : this.listeners)

			listener.removed(new ExpiringHashMapEntry((K)key, value));

		return value;

			listener.removed(new ExpiringHashMapEntry((K)key, evalue.getValue()));

		return evalue.getValue();

	}

	

	@Override
@@ -135,28 +149,49 @@ public class LRUExpiringHashMap<K extends Serializable, V> implements ListeningM 
	}

	

	public void exipriationCheck() {

		Iterator<Entry<ExpiringHashKey, V>> i = this.map.entrySet().iterator();

		for (Entry<ExpiringHashKey, V> entry = i.next(); i.hasNext(); entry = i.next()) {

			if (entry.getKey().isExpired()) {

		// since these should be order, we could break out of this loop once we reach an unexpired entry

		List<ExpiringHashMapEntry> entries = new LinkedList<>();

		

		synchronized (this.sync) {

			Iterator<Entry<K, ExpiringValue>> i = this.map.entrySet().iterator();

			for (Entry<K, ExpiringValue> entry = i.next(); i.hasNext(); entry = i.next()) {

				if (entry.getValue().isExpired()) {

					i.remove();

				for (MapListener<K, V> listener : this.listeners)

					if (listener instanceof ExpiringMapListener)

						((ExpiringMapListener<K, V>)listener).expired(new ExpiringHashMapEntry(entry.getKey().getKey(), entry.getValue()));

					entries.add(new ExpiringHashMapEntry(entry.getKey(), entry.getValue().getValue()));

				} else {

					// ordered map; ordered by time of entry

					// since expiration timers are constant, we can skip looping

					break;

				}

			}

		}



		for (ExpiringHashMapEntry entry : entries)

			for (MapListener<K, V> listener : this.listeners)

				if (listener instanceof ExpiringMapListener)

					((ExpiringMapListener<K, V>)listener).expired(entry);

	}

	

	public void expire(K key) {

    	if (this.logger.isDebugEnabled())

    		this.logger.debug("expiring key from map: " + key);



		ExpiringValue evalue;

		

    	synchronized (this.sync) {

			if (!this.map.containsKey(key))

				return;

			evalue = this.map.remove(key);

    	}

    	

		V value = this.map.remove(key);

		for (MapListener<K, V> listener : this.listeners)

			if (listener instanceof ExpiringMapListener)

				((ExpiringMapListener<K, V>)listener).expired(new ExpiringHashMapEntry(key, value));

				((ExpiringMapListener<K, V>)listener).expired(new ExpiringHashMapEntry(key, evalue.getValue()));

	}

	

	@Override

	public String toString() {

		return this.map.toString();

	}
@@ -207,44 +242,27 @@ public class LRUExpiringHashMap<K extends Serializable, V> implements ListeningM 
		

	}

	

	private class ExpiringHashKey implements Serializable {

	

		private static final long serialVersionUID = -6511298315143655313L;

	private class ExpiringValue {

	

		private long expirationTimeInMillis;

		private K key;

		private V value;

		

		public ExpiringHashKey(K key, long expirationTimeInMillis) {

			this.key = key;

		public ExpiringValue(V value, long expirationTimeInMillis) {

			this.value = value;

			this.expirationTimeInMillis = expirationTimeInMillis;

		}

		

		public K getKey() {

			return this.key;

		public V getValue() {

			return this.value;

		}

		

		public boolean isExpired() {

			return this.expirationTimeInMillis <= System.currentTimeMillis();

		}

		

		@Override

		public int hashCode() {

			return this.key.hashCode();

		}

		

		@Override

		public String toString() {

			return this.key.toString();

		}

		

		@Override

		@SuppressWarnings("unchecked")

		public boolean equals(Object obj) {

			if (obj instanceof LRUExpiringHashMap.ExpiringHashKey) {

				return this.key.equals(((ExpiringHashKey)obj).key);

			} else {

				return this.key.equals(obj);

			}

			return this.value.toString();

		}

		

	}

src/main/java/com/inteligr8/git/LocalRepositoryCache.java

+111 −55
Original line number Diff line number Diff line
@@ -2,14 +2,20 @@ package com.inteligr8.git; 


import java.io.File;

import java.io.IOException;

import java.net.URISyntaxException;

import java.util.HashMap;

import java.util.Map;

import java.util.Map.Entry;

import java.util.UUID;

import java.util.concurrent.Semaphore;



import org.eclipse.jgit.api.CloneCommand;

import org.eclipse.jgit.api.Git;

import org.eclipse.jgit.api.ResetCommand.ResetType;

import org.eclipse.jgit.api.errors.GitAPIException;

import org.eclipse.jgit.api.errors.InvalidRemoteException;

import org.eclipse.jgit.api.errors.RefNotFoundException;

import org.eclipse.jgit.api.errors.TransportException;

import org.eclipse.jgit.transport.CredentialsProvider;

import org.eclipse.jgit.util.FileUtils;

import org.slf4j.Logger;
@@ -23,15 +29,18 @@ public class LocalRepositoryCache { 
		return INSTANCE;

	}

	

	

	

	private final Logger logger = LoggerFactory.getLogger(LocalRepositoryCache.class);

	

	private final LRUExpiringHashMap<String, Git> cachedGits = new LRUExpiringHashMap<String, Git>(30);

//	private final Map<String, String> gitIdsToUrls = new HashMap<String, String>();

	private final File cacheDirectory = new File(System.getProperty("java.io.tmpdir"), "git");

	private final LRUExpiringHashMap<String, CachedGit> cachedGits = new LRUExpiringHashMap<>(30);

	private final Map<String, Semaphore> gitUrlSemaphores = new HashMap<>();

	final File cacheDirectory = new File(System.getProperty("java.io.tmpdir"), "git");

	private final int simultaneousProcessesPerGitRepo = 1;

	

	private LocalRepositoryCache() {

		this.cacheDirectory.mkdir();

		this.cachedGits.addListener(new RepositoryCacheMapListener());

		this.cachedGits.addListener(new RepositoryCacheMapListener<CachedGit>());

	}

	

	@Override
@@ -49,27 +58,49 @@ public class LocalRepositoryCache { 
		this.clear();

	}

	

	public synchronized ExtendedGit acquire(String url) throws GitAPIException, InvalidRemoteException, RefNotFoundException {

	public CachedGit acquire(String url) throws InterruptedException, InvalidRemoteException, RefNotFoundException, TransportException, GitAPIException {

		return this.acquire(url, null, null);

	}

	

	public synchronized ExtendedGit acquire(String url, CredentialsProvider creds) throws GitAPIException, InvalidRemoteException, RefNotFoundException {

	public CachedGit acquire(String url, CredentialsProvider creds) throws InterruptedException, InvalidRemoteException, RefNotFoundException, TransportException, GitAPIException {

		return this.acquire(url, creds, null);

	}

	

	public synchronized ExtendedGit acquire(String url, String branch) throws GitAPIException, InvalidRemoteException, RefNotFoundException {

	public CachedGit acquire(String url, String branch) throws InterruptedException, InvalidRemoteException, RefNotFoundException, TransportException, GitAPIException {

		return this.acquire(url, null, branch);

	}

	

	public synchronized ExtendedGit acquire(String url, CredentialsProvider creds, String branch) throws GitAPIException, InvalidRemoteException, RefNotFoundException {

	/**

	 * 

	 * @param url A URL for the Git Repository.

	 * @param creds A username/password for the Git Repository.

	 * @param branch A branch name of the branch in the Git Repository to start working on.

	 * @return A JGit object.

	 * @throws InterruptedException The JVM was interrupted while the thread was waiting for another request on the same Git Repository

	 * @throws InvalidRemoteException The URL refers to nothing or an invalid Git Repository

	 * @throws RefNotFoundException The branch does not exist in the Git Repository represented by the URL

	 * @throws TransportException A disk or network error or hiccup occurred.

	 * @throws GitAPIException A Git API, library, or protocol related issue occurred

	 */

	public CachedGit acquire(String url, CredentialsProvider creds, String branch)

	throws InterruptedException, InvalidRemoteException, RefNotFoundException, TransportException, GitAPIException {

    	if (this.logger.isTraceEnabled())

    		this.logger.trace("acquire('" + url + "', " + creds + ", '" + branch + "')");

    	

//		Git git = this.cachedGits.remove(url);

//		if (git == null) {

	    	if (this.logger.isDebugEnabled())

	    		this.logger.debug("creating temporary Git directory");

    	Semaphore semaphore = null;

    	synchronized (this) {

	    	semaphore = this.gitUrlSemaphores.get(url);

	    	if (semaphore == null)

	    		this.gitUrlSemaphores.put(url, semaphore = new Semaphore(this.simultaneousProcessesPerGitRepo));

    	}

    	

    	semaphore.acquire();

    	try {

			CachedGit git = this.cachedGits.remove(url);

			if (git == null) {

				File gitRepoDirectory = new File(this.cacheDirectory, UUID.randomUUID().toString() + ".git");

		    	if (this.logger.isDebugEnabled())

		    		this.logger.debug("Git directory cache for clone: " + gitRepoDirectory);

				

				CloneCommand clone = new CloneCommand()

		    			.setURI(url)
@@ -78,45 +109,72 @@ public class LocalRepositoryCache { 
					clone.setBranch(branch);

				

		    	if (this.logger.isDebugEnabled())

	    		this.logger.debug("cloning Git repository: " + url);

	    	ExtendedGit git = creds != null ? new CredentialedGit(clone, creds) : new ExtendedGit(clone);

		    		this.logger.debug("Cloning Git repository: " + url);

		    	git = creds != null ? new CredentialedGit(clone, creds) : new ExtendedGit(clone);

		    	if (this.logger.isInfoEnabled())

	    		this.logger.info("Cloned Git Repository");

		    		this.logger.info("Cloned Git Repository: " + ((ExtendedGit)git).getRepositoryFullyQualifiedName());

			} else {

		    	if (this.logger.isDebugEnabled())

		    		this.logger.debug("reseting Git");

				git.reset().setMode(ResetType.HARD).call();

				

				if (branch != null) {

			    	if (this.logger.isDebugEnabled())

			    		this.logger.debug("switching Git branches: " + branch);

					git.checkout().setName(branch).call();

				}

	

		    	if (this.logger.isDebugEnabled())

		    		this.logger.debug("updating Git branch: " + git.getRepository().getBranch());

				git.pull().call();

			}



	    	return git;

//			git = creds != null ? new CredentialedGit(clone, creds) : new CachedGit(clone);

//			this.gitIdsToUrls.put(git.getRepository().getIdentifier(), url);

//		} else {

//			if (branch != null) {

//		    	if (this.logger.isDebugEnabled())

//		    		this.logger.debug("switching Git branches: " + branch);

//				git.checkout().setName(branch).call();

//			}

//

//	    	if (this.logger.isDebugEnabled())

//	    		this.logger.debug("updating Git branch: " + branch);

//			git.pull().call();

//		}

	}

	

	public synchronized void release(Git git) {

    	} catch (URISyntaxException use) {

    		semaphore.release();

    		throw new DeveloperException(this.logger, use);

    	} catch (IOException ie) {

    		semaphore.release();

    		throw new TransportException("A I/O issue occurred", ie);

    	} catch (GitAPIException gae) {

    		semaphore.release();

    		throw gae;

    	}

	}

	

	public void release(CachedGit git) {

		if (this.logger.isTraceEnabled())

    		this.logger.trace("release('" + git.getRepository().getIdentifier() + "')");

		

//		String url = this.gitIdsToUrls.get(git.getRepository().getIdentifier());

//		this.cachedGits.put(url, git);

		this.expunge(git);

		try {

			String url = git.getFirstRemoteUrl();

			

			synchronized (this) {

				Semaphore semaphore = this.gitUrlSemaphores.get(url);

				if (semaphore != null)

					semaphore.release();

			}

			

	public synchronized void clear() {

			this.cachedGits.put(url, git);

		} catch (GitAPIException gae) {

			if (this.logger.isDebugEnabled())

				this.logger.debug("A Git repository was never released", gae);

			this.logger.warn("A Git repository was never released, potentially blocking all future processing by this tool until a restart: " + git.toString());

		}

	}

	

	public void clear() {

		if (this.logger.isTraceEnabled())

    		this.logger.trace("clear()");

		

//		this.cachedGits.clear();

		this.cachedGits.clear();

		

		for (Semaphore semaphore : this.gitUrlSemaphores.values())

			; // FIXME interrupt all the threads; releasing semaphores will not be enough

		this.gitUrlSemaphores.clear();

	}

	

	private void expunge(Git git) {

//		gitIdsToUrls.remove(git.getRepository().getIdentifier());

		

		File gitDir = git.getRepository().getDirectory();

		File workingTreeDir = git.getRepository().getWorkTree();

		git.getRepository().close();
@@ -129,7 +187,6 @@ public class LocalRepositoryCache { 
			this.logger.warn("Failed to delete a git directory: " + gitDir);

			if (this.logger.isDebugEnabled())

				this.logger.debug(ie.getMessage(), ie);

			gitDir.deleteOnExit();

		}



		try {
@@ -140,42 +197,41 @@ public class LocalRepositoryCache { 
			this.logger.warn("Failed to delete a git directory: " + workingTreeDir);

			if (this.logger.isDebugEnabled())

				this.logger.debug(ie.getMessage(), ie);

			workingTreeDir.deleteOnExit();

		}

		

		if (this.logger.isInfoEnabled())

			this.logger.info("Deleted Git Repository");

			this.logger.info("Deleted Git Repository: " + gitDir);

	}

	

	

	

	private class RepositoryCacheMapListener implements ExpiringMapListener<String, Git> {

	private class RepositoryCacheMapListener<T extends CachedGit> implements ExpiringMapListener<String, T> {



		private final Logger logger = LoggerFactory.getLogger(LocalRepositoryCache.class);

		

		@Override

		public void accessed(Entry<String, Git> entry) {

		public void accessed(Entry<String, T> entry) {

		}

		

		@Override

		public void added(Entry<String, Git> entry) {

		public void added(Entry<String, T> entry) {

			// a clean one or one returned after being previously removed

		}

		

		@Override

		public void expired(Entry<String, Git> entry) {

		public void expired(Entry<String, T> entry	) {

			if (this.logger.isTraceEnabled())

	    		this.logger.trace("expired('" + entry.getKey() + "', '" + entry.getValue().getRepository().getIdentifier() + "')");

			expunge(entry.getValue());

		}

		

		@Override

		public void removed(Entry<String, Git> entry) {

		public void removed(Entry<String, T> entry) {

			// expected to be removed only temporarily...for use elsewhere; do not close

		}

		

		@Override

		public void cleared(Entry<String, Git> entry) {

		public void cleared(Entry<String, T> entry) {

			if (this.logger.isTraceEnabled())

	    		this.logger.trace("cleared('" + entry.getKey() + "', '" + entry.getValue().getRepository().getIdentifier() + "')");

			expunge(entry.getValue());