From 6b23332e9def8fc0e75be883f8a18807f0fccbbd Mon Sep 17 00:00:00 2001 From: Olivier Lamy Date: Thu, 12 Dec 2013 11:26:40 +0000 Subject: add a new module for file locking git-svn-id: https://svn.apache.org/repos/asf/archiva/trunk@1550396 13f79535-47bb-0310-9956-ffa450edef68 --- .../archiva-base/archiva-configuration/pom.xml | 80 ++++---- .../archiva-base/archiva-filelock/pom.xml | 134 +++++++++++++ .../common/filelock/DefaultFileLockManager.java | 177 ++++++++++++++++++ .../archiva/common/filelock/FileLockException.java | 13 ++ .../archiva/common/filelock/FileLockManager.java | 18 ++ .../org/apache/archiva/common/filelock/Lock.java | 105 +++++++++++ .../src/main/resources/META-INF/spring-context.xml | 34 ++++ .../src/test/cassandra-all-2.0.3.jar | Bin 0 -> 3369436 bytes .../filelock/DefaultFileLockManagerTest.java | 208 +++++++++++++++++++++ .../src/test/resources/log4j2-test.xml | 44 +++++ .../proxy/DefaultRepositoryProxyConnectors.java | 4 + archiva-modules/archiva-base/pom.xml | 1 + 12 files changed, 783 insertions(+), 35 deletions(-) create mode 100644 archiva-modules/archiva-base/archiva-filelock/pom.xml create mode 100644 archiva-modules/archiva-base/archiva-filelock/src/main/java/org/apache/archiva/common/filelock/DefaultFileLockManager.java create mode 100644 archiva-modules/archiva-base/archiva-filelock/src/main/java/org/apache/archiva/common/filelock/FileLockException.java create mode 100644 archiva-modules/archiva-base/archiva-filelock/src/main/java/org/apache/archiva/common/filelock/FileLockManager.java create mode 100644 archiva-modules/archiva-base/archiva-filelock/src/main/java/org/apache/archiva/common/filelock/Lock.java create mode 100644 archiva-modules/archiva-base/archiva-filelock/src/main/resources/META-INF/spring-context.xml create mode 100644 archiva-modules/archiva-base/archiva-filelock/src/test/cassandra-all-2.0.3.jar create mode 100644 archiva-modules/archiva-base/archiva-filelock/src/test/java/org/apache/archiva/common/filelock/DefaultFileLockManagerTest.java create mode 100644 archiva-modules/archiva-base/archiva-filelock/src/test/resources/log4j2-test.xml (limited to 'archiva-modules/archiva-base') diff --git a/archiva-modules/archiva-base/archiva-configuration/pom.xml b/archiva-modules/archiva-base/archiva-configuration/pom.xml index 94d3fa219..577e3a884 100644 --- a/archiva-modules/archiva-base/archiva-configuration/pom.xml +++ b/archiva-modules/archiva-base/archiva-configuration/pom.xml @@ -39,6 +39,16 @@ + + org.springframework + spring-context + + + commons-logging + commons-logging + + + org.apache.archiva.redback.components.registry spring-registry-commons @@ -52,41 +62,41 @@ commons-configuration commons-configuration - - - commons-beanutils - commons-beanutils-core - - - - xerces - xerces - - - xerces - xercesImpl - - - xalan - xalan - - - xml-apis - xml-apis - - - servletapi - servletapi - - - commons-logging - commons-logging - - - ant - ant-optional - - + + + commons-beanutils + commons-beanutils-core + + + + xerces + xerces + + + xerces + xercesImpl + + + xalan + xalan + + + xml-apis + xml-apis + + + servletapi + servletapi + + + commons-logging + commons-logging + + + ant + ant-optional + + org.apache.ant diff --git a/archiva-modules/archiva-base/archiva-filelock/pom.xml b/archiva-modules/archiva-base/archiva-filelock/pom.xml new file mode 100644 index 000000000..d8ab67df0 --- /dev/null +++ b/archiva-modules/archiva-base/archiva-filelock/pom.xml @@ -0,0 +1,134 @@ + + + + + + org.apache.archiva + archiva-base + 2.0.0-SNAPSHOT + + 4.0.0 + archiva-filelock + bundle + Archiva Base :: FileLock + + + + org.springframework + spring-context + + + commons-logging + commons-logging + + + + + + javax.inject + javax.inject + + + + org.slf4j + slf4j-api + + + + org.apache.logging.log4j + log4j-jcl + test + + + + com.googlecode.multithreadedtc + multithreadedtc + 1.01 + test + + + + org.apache.logging.log4j + log4j-slf4j-impl + test + + + + org.apache.logging.log4j + log4j-core + test + + + + commons-io + commons-io + test + + + + org.springframework + spring-test + test + + + + + + + org.apache.felix + maven-bundle-plugin + + + org.apache.archiva.common + ${project.version} + + org.apache.archiva.common*;version=${project.version} + + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + ${project.build.directory} + ${project.build.directory} + + + + + + + + org.apache.rat + apache-rat-plugin + + + + + + + + + diff --git a/archiva-modules/archiva-base/archiva-filelock/src/main/java/org/apache/archiva/common/filelock/DefaultFileLockManager.java b/archiva-modules/archiva-base/archiva-filelock/src/main/java/org/apache/archiva/common/filelock/DefaultFileLockManager.java new file mode 100644 index 000000000..795760672 --- /dev/null +++ b/archiva-modules/archiva-base/archiva-filelock/src/main/java/org/apache/archiva/common/filelock/DefaultFileLockManager.java @@ -0,0 +1,177 @@ +package org.apache.archiva.common.filelock; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import java.io.File; +import java.io.IOException; + +import java.nio.channels.OverlappingFileLockException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * @author Olivier Lamy + */ +@Service("fileLockManager#default") +public class DefaultFileLockManager + implements FileLockManager +{ + private static final ConcurrentMap lockFiles = new ConcurrentHashMap( 64 ); + + private boolean skipLocking = false; + + private Logger log = LoggerFactory.getLogger( getClass() ); + + @Override + public Lock readFileLock( File file ) + throws FileLockException + { + if ( skipLocking ) + { + try + { + return new Lock( file, false ); + } + catch ( IOException e ) + { + throw new FileLockException( e.getMessage(), e ); + } + } + Lock lock = lockFiles.get( file ); + if ( lock == null ) + { + try + { + lock = new Lock( file, false ); + Lock current = lockFiles.putIfAbsent( file, lock ); + if ( current != null ) + { + lock = current; + } + return lock; + } + catch ( IOException e ) + { + throw new FileLockException( e.getMessage(), e ); + } + catch ( OverlappingFileLockException e ) + { + log.debug( "OverlappingFileLockException: {}", e.getMessage() ); + if ( lock == null ) + { + lock = lockFiles.get( file ); + } + } + } + // FIXME add a timeout on getting that!!! + while ( true ) + { + log.debug( "wait read lock" ); + synchronized ( lock ) + { + if ( lock.getFileLock().isShared() || !lock.getFileLock().isValid() ) + { + lock.addFileClient( Thread.currentThread() ); + return lock; + } + } + } + //return lock; + } + + @Override + public Lock writeFileLock( File file ) + throws FileLockException + { + try + { + if ( skipLocking ) + { + return new Lock( file, true ); + } + + // FIXME add a timeout on getting that!!! + while ( true ) + { + Lock lock = lockFiles.get( file ); + log.debug( "wait write lock" ); + if ( lock != null ) + { + synchronized ( lock ) + { + if ( lock.getFileLock().isValid() || lock.getFileClients().size() > 0 ) + { + continue; + } + return lock; + } + } + else + { + try + { + lock = new Lock( file, true ); + } + catch ( OverlappingFileLockException e ) + { + log.debug( "OverlappingFileLockException: {}", e.getMessage() ); + if ( lock == null ) + { + lock = lockFiles.get( file ); + } + + lock = lockFiles.get( file ); + log.debug( "OverlappingFileLockException get: {}", lock ); + } + Lock current = lockFiles.putIfAbsent( file, lock ); + if ( current != null ) + { + lock = current; + } + return lock; + } + } + + } + catch ( IOException e ) + { + throw new FileLockException( e.getMessage(), e ); + } + } + + @Override + public void release( Lock lock ) + throws FileLockException + { + if ( lock == null ) + { + log.debug( "skip releasing null" ); + return; + } + if ( skipLocking ) + { + return; + } + try + { + if ( lock.isWrite().get() ) + { + lock.getFileLock().release(); + } + synchronized ( lock ) + { + lock.close(); + if ( lock.getFileClients().size() < 1 ) + { + lockFiles.remove( lock.getFile() ); + } + } + } + catch ( IOException e ) + { + throw new FileLockException( e.getMessage(), e ); + } + } +} diff --git a/archiva-modules/archiva-base/archiva-filelock/src/main/java/org/apache/archiva/common/filelock/FileLockException.java b/archiva-modules/archiva-base/archiva-filelock/src/main/java/org/apache/archiva/common/filelock/FileLockException.java new file mode 100644 index 000000000..2d14859c0 --- /dev/null +++ b/archiva-modules/archiva-base/archiva-filelock/src/main/java/org/apache/archiva/common/filelock/FileLockException.java @@ -0,0 +1,13 @@ +package org.apache.archiva.common.filelock; + +/** + * @author Olivier Lamy + */ +public class FileLockException + extends Exception +{ + public FileLockException( String s, Throwable throwable ) + { + super( s, throwable ); + } +} diff --git a/archiva-modules/archiva-base/archiva-filelock/src/main/java/org/apache/archiva/common/filelock/FileLockManager.java b/archiva-modules/archiva-base/archiva-filelock/src/main/java/org/apache/archiva/common/filelock/FileLockManager.java new file mode 100644 index 000000000..5f3832634 --- /dev/null +++ b/archiva-modules/archiva-base/archiva-filelock/src/main/java/org/apache/archiva/common/filelock/FileLockManager.java @@ -0,0 +1,18 @@ +package org.apache.archiva.common.filelock; + +import java.io.File; + +/** + * @author Olivier Lamy + */ +public interface FileLockManager +{ + Lock writeFileLock( File file ) + throws FileLockException; + + Lock readFileLock( File file ) + throws FileLockException; + + void release( Lock lock ) + throws FileLockException; +} diff --git a/archiva-modules/archiva-base/archiva-filelock/src/main/java/org/apache/archiva/common/filelock/Lock.java b/archiva-modules/archiva-base/archiva-filelock/src/main/java/org/apache/archiva/common/filelock/Lock.java new file mode 100644 index 000000000..6baabbc05 --- /dev/null +++ b/archiva-modules/archiva-base/archiva-filelock/src/main/java/org/apache/archiva/common/filelock/Lock.java @@ -0,0 +1,105 @@ +package org.apache.archiva.common.filelock; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.channels.FileLock; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * @author Olivier Lamy + */ +public class Lock +{ + private File file; + + private AtomicBoolean write; + + private final Map fileClients = new HashMap(); + + private FileLock fileLock; + + public Lock( File file, boolean write ) + throws FileNotFoundException, IOException + { + this.file = file; + this.write = new AtomicBoolean( write ); + this.openLock( write ); + } + + public File getFile() + { + return file; + } + + public AtomicBoolean isWrite() + { + return write; + } + + public void setFile( File file ) + { + this.file = file; + } + + public void setWrite( boolean write ) + { + this.write.set( write ); + } + + public FileLock getFileLock() + { + return fileLock; + } + + public void setFileLock( FileLock fileLock ) + { + this.fileLock = fileLock; + } + + public Map getFileClients() + { + return fileClients; + } + + public void addFileClient( Thread thread ) + { + this.fileClients.put( thread, new AtomicInteger( 1 ) ); + } + + public boolean removeFileClient( Thread thread ) + { + return this.fileClients.remove( thread ) != null; + } + + protected void close() + throws IOException + { + if ( this.write.get() ) + { + this.fileLock.release(); + fileClients.remove( Thread.currentThread() ); + } + } + + public void openLock( boolean write ) + throws IOException + { + fileClients.put( Thread.currentThread(), new AtomicInteger( 1 ) ); + RandomAccessFile raf = new RandomAccessFile( file, write ? "rw" : "r" ); + this.fileLock = raf.getChannel().lock( 1, 1, !write ); + } + + @Override + public String toString() + { + final StringBuilder sb = new StringBuilder( "Lock{" ); + sb.append( "file=" ).append( file ); + sb.append( '}' ); + return sb.toString(); + } +} diff --git a/archiva-modules/archiva-base/archiva-filelock/src/main/resources/META-INF/spring-context.xml b/archiva-modules/archiva-base/archiva-filelock/src/main/resources/META-INF/spring-context.xml new file mode 100644 index 000000000..944c8d606 --- /dev/null +++ b/archiva-modules/archiva-base/archiva-filelock/src/main/resources/META-INF/spring-context.xml @@ -0,0 +1,34 @@ + + + + + + + + + + \ No newline at end of file diff --git a/archiva-modules/archiva-base/archiva-filelock/src/test/cassandra-all-2.0.3.jar b/archiva-modules/archiva-base/archiva-filelock/src/test/cassandra-all-2.0.3.jar new file mode 100644 index 000000000..0a4b1c542 Binary files /dev/null and b/archiva-modules/archiva-base/archiva-filelock/src/test/cassandra-all-2.0.3.jar differ diff --git a/archiva-modules/archiva-base/archiva-filelock/src/test/java/org/apache/archiva/common/filelock/DefaultFileLockManagerTest.java b/archiva-modules/archiva-base/archiva-filelock/src/test/java/org/apache/archiva/common/filelock/DefaultFileLockManagerTest.java new file mode 100644 index 000000000..86fb24dd3 --- /dev/null +++ b/archiva-modules/archiva-base/archiva-filelock/src/test/java/org/apache/archiva/common/filelock/DefaultFileLockManagerTest.java @@ -0,0 +1,208 @@ +package org.apache.archiva.common.filelock; + +import edu.umd.cs.mtc.MultithreadedTestCase; +import edu.umd.cs.mtc.TestFramework; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import javax.inject.Inject; +import javax.inject.Named; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; + +/** + * @author Olivier Lamy + */ +@RunWith( SpringJUnit4ClassRunner.class ) +@ContextConfiguration( locations = { "classpath*:/META-INF/spring-context.xml" } ) +public class DefaultFileLockManagerTest +{ + + final Logger logger = LoggerFactory.getLogger( getClass() ); + + @Inject + @Named( value = "fileLockManager#default" ) + FileLockManager fileLockManager; + + class ConcurentFileWrite + extends MultithreadedTestCase + { + + FileLockManager fileLockManager; + + File file = new File( System.getProperty( "buildDirectory" ), "foo.txt" ); + + File largeJar = new File( System.getProperty( "basedir" ), "src/test/cassandra-all-2.0.3.jar" ); + + ConcurentFileWrite( FileLockManager fileLockManager ) + throws IOException + { + this.fileLockManager = fileLockManager; + file.createNewFile(); + + } + + @Override + public void initialize() + { + + } + + public void thread1() + throws FileLockException, IOException + { + logger.info( "thread1" ); + Lock lock = fileLockManager.writeFileLock( this.file ); + try + { + lock.getFile().delete(); + FileUtils.copyFile( largeJar, lock.getFile() ); + } + finally + { + fileLockManager.release( lock ); + } + logger.info( "thread1 ok" ); + } + + public void thread2() + throws FileLockException, IOException + { + logger.info( "thread2" ); + Lock lock = fileLockManager.writeFileLock( this.file ); + try + { + lock.getFile().delete(); + FileUtils.copyFile( largeJar, lock.getFile() ); + } + finally + { + fileLockManager.release( lock ); + } + logger.info( "thread2 ok" ); + } + + public void thread3() + throws FileLockException, IOException + { + logger.info( "thread3" ); + Lock lock = fileLockManager.readFileLock( this.file ); + try + { + IOUtils.copy( new FileInputStream( lock.getFile() ), + new FileOutputStream( File.createTempFile( "foo", ".jar" ) ) ); + } + finally + { + fileLockManager.release( lock ); + } + logger.info( "thread3 ok" ); + } + + public void thread4() + throws FileLockException, IOException + { + logger.info( "thread4" ); + Lock lock = fileLockManager.writeFileLock( this.file ); + try + { + lock.getFile().delete(); + FileUtils.copyFile( largeJar, lock.getFile() ); + } + finally + { + fileLockManager.release( lock ); + } + logger.info( "thread4 ok" ); + } + + public void thread5() + throws FileLockException, IOException + { + logger.info( "thread5" ); + Lock lock = fileLockManager.writeFileLock( this.file ); + try + { + lock.getFile().delete(); + FileUtils.copyFile( largeJar, lock.getFile() ); + } + finally + { + fileLockManager.release( lock ); + } + logger.info( "thread5 ok" ); + } + + public void thread6() + throws FileLockException, IOException + { + logger.info( "thread6" ); + Lock lock = fileLockManager.readFileLock( this.file ); + try + { + IOUtils.copy( new FileInputStream( lock.getFile() ), + new FileOutputStream( File.createTempFile( "foo", ".jar" ) ) ); + } + finally + { + fileLockManager.release( lock ); + } + logger.info( "thread6 ok" ); + } + + public void thread7() + throws FileLockException, IOException + { + logger.info( "thread7" ); + Lock lock = fileLockManager.writeFileLock( this.file ); + try + { + lock.getFile().delete(); + FileUtils.copyFile( largeJar, lock.getFile() ); + } + finally + { + fileLockManager.release( lock ); + } + logger.info( "thread7 ok" ); + } + + public void thread8() + throws FileLockException, IOException + { + logger.info( "thread8" ); + Lock lock = fileLockManager.readFileLock( this.file ); + try + { + IOUtils.copy( new FileInputStream( lock.getFile() ), + new FileOutputStream( File.createTempFile( "foo", ".jar" ) ) ); + } + finally + { + fileLockManager.release( lock ); + } + logger.info( "thread8 ok" ); + } + + + + } + + @Test + public void testWrite() + throws Throwable + { + ConcurentFileWrite concurentFileWrite = new ConcurentFileWrite( fileLockManager ); + //concurentFileWrite.setTrace( true ); + TestFramework.runOnce( concurentFileWrite ); + } + +} diff --git a/archiva-modules/archiva-base/archiva-filelock/src/test/resources/log4j2-test.xml b/archiva-modules/archiva-base/archiva-filelock/src/test/resources/log4j2-test.xml new file mode 100644 index 000000000..93e519df7 --- /dev/null +++ b/archiva-modules/archiva-base/archiva-filelock/src/test/resources/log4j2-test.xml @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/archiva-modules/archiva-base/archiva-proxy/src/main/java/org/apache/archiva/proxy/DefaultRepositoryProxyConnectors.java b/archiva-modules/archiva-base/archiva-proxy/src/main/java/org/apache/archiva/proxy/DefaultRepositoryProxyConnectors.java index 9b78010e6..72eb22e8e 100644 --- a/archiva-modules/archiva-base/archiva-proxy/src/main/java/org/apache/archiva/proxy/DefaultRepositoryProxyConnectors.java +++ b/archiva-modules/archiva-base/archiva-proxy/src/main/java/org/apache/archiva/proxy/DefaultRepositoryProxyConnectors.java @@ -83,6 +83,7 @@ import javax.inject.Inject; import javax.inject.Named; import java.io.File; import java.io.IOException; +import java.io.RandomAccessFile; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -1091,6 +1092,9 @@ public class DefaultRepositoryProxyConnectors } target.getParentFile().mkdirs(); + // TODO file lock library + RandomAccessFile raf; + if ( !temp.renameTo( target ) ) { log.warn( "Unable to rename tmp file to its final name... resorting to copy command." ); diff --git a/archiva-modules/archiva-base/pom.xml b/archiva-modules/archiva-base/pom.xml index d8d8d7748..429ee5257 100644 --- a/archiva-modules/archiva-base/pom.xml +++ b/archiva-modules/archiva-base/pom.xml @@ -32,6 +32,7 @@ archiva-test-utils archiva-common + archiva-filelock archiva-model archiva-configuration archiva-checksum -- cgit v1.2.3