Improving checksum implementation. Adding multiple algorithms.

archiva-modules/archiva-base/archiva-checksum/src/main/java/org/apache/archiva/checksum/Checksum.java

@@ -19,60 +19,22 @@ package org.apache.archiva.checksum;
  * under the License.
  */
 
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.io.output.NullOutputStream;
-
-import java.io.FileNotFoundException;
 import java.io.IOException;
-import java.io.InputStream;
 import java.nio.ByteBuffer;
 import java.nio.MappedByteBuffer;
 import java.nio.channels.FileChannel;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
-import java.security.DigestInputStream;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
-import java.util.List;
 
 /**
  * Checksum - simple checksum hashing routines.
  */
 public class Checksum
 {
-    private static final int BUFFER_SIZE = 32768;
     private byte[] result = new byte[0];
 
-    public static void update( List<Checksum> checksums, Path file )
-        throws ChecksumValidationException
-    {
-        long fileSize;
-        try (FileChannel channel = FileChannel.open(file, StandardOpenOption.READ )) {
-            fileSize = channel.size();
-            long pos = 0;
-            while (pos<fileSize)
-            {
-                long bufferSize = Math.min(BUFFER_SIZE, fileSize-pos);
-                MappedByteBuffer buffer = channel.map( FileChannel.MapMode.READ_ONLY, pos, bufferSize);
-                for ( Checksum checksum : checksums )
-                {
-                    checksum.update( buffer );
-                    buffer.rewind();
-                }
-                fileSize = channel.size();
-                pos += BUFFER_SIZE;
-            }
-            for (Checksum checksum : checksums) {
-                checksum.finish();
-            }
-        } catch(FileNotFoundException e)
-        {
-            throw new ChecksumValidationException( ChecksumValidationException.ValidationError.FILE_NOT_FOUND, "File that should be parsed, not found: "+e.getMessage(), e );
-        } catch(IOException e) {
-            throw new ChecksumValidationException( ChecksumValidationException.ValidationError.READ_ERROR, "Parsing of file failed: "+e.getMessage(), e );
-        }
-    }
-
     private final MessageDigest md;
 
     private ChecksumAlgorithm checksumAlgorithm;
@@ -142,26 +104,6 @@ public class Checksum
         return this;
     }
 
-    public void update( Path file )
-        throws IOException
-    {
-        long fileSize;
-        try (FileChannel channel = FileChannel.open(file, StandardOpenOption.READ )) {
-            fileSize = channel.size();
-            long pos = 0;
-            while (pos<fileSize)
-            {
-                long bufferSize = Math.min(BUFFER_SIZE, fileSize-pos);
-                MappedByteBuffer buffer = channel.map( FileChannel.MapMode.READ_ONLY, pos, bufferSize);
-                update( buffer );
-                buffer.rewind();
-                fileSize = channel.size();
-                pos += BUFFER_SIZE;
-            }
-            finish();
-        }
-    }
-
     public boolean compare(byte[] cmp) {
         if (this.result == null || this.result.length==0) {
             finish();

archiva-modules/archiva-base/archiva-checksum/src/main/java/org/apache/archiva/checksum/ChecksumAlgorithm.java

@@ -35,14 +35,18 @@ import java.util.Set;
 /**
  * Enumeration of available ChecksumAlgorithm techniques.
  *
+ * Each algorithm represents a message digest algorithm and has a unique type.
+ * The type string may be used in the hash files (FreeBSD and OpenSSL add the type to the hash file)
+ *
+ * There are multiple file extensions. The first one is considered the default extension.
  *
  */
 public enum ChecksumAlgorithm {
     MD5("MD5", "MD5", "md5"),
     SHA1("SHA-1", "SHA1", "sha1", "sha128", "sha-128"),
-    SHA256("SHA-256", "SHA2", "sha2", "sha256", "sha-256"),
-    SHA384("SHA-384", "SHA3", "sha3", "sha384", "sha-384"),
-    SHA512("SHA-512", "SHA5", "sha5", "sha512", "sha-512");
+    SHA256("SHA-256", "SHA256", "sha256", "sha2", "sha-256"),
+    SHA384("SHA-384", "SHA384", "sha384", "sha3", "sha-384"),
+    SHA512("SHA-512", "SHA512", "sha512", "sha5", "sha-512");
 
     public static ChecksumAlgorithm getByExtension( Path file )
     {
@@ -59,12 +63,12 @@ public enum ChecksumAlgorithm {
         for (ChecksumAlgorithm alg : ChecksumAlgorithm.values()) {
             for (String extString : alg.getExt())
             {
-                extensionMap.put( extString, alg );
+                extensionMap.put( extString.toLowerCase(), alg );
             }
         }
     }
 
-    public static Set<String> getExtensions() {
+    public static Set<String> getAllExtensions() {
         return extensionMap.keySet();
     }
 
@@ -74,7 +78,7 @@ public enum ChecksumAlgorithm {
     private final String algorithm;
 
     /**
-     * The file extension for this ChecksumAlgorithm.
+     * The file extensions for this ChecksumAlgorithm.
      */
     private final List<String> ext;
 
@@ -87,8 +91,8 @@ public enum ChecksumAlgorithm {
      * Construct a ChecksumAlgorithm
      * 
      * @param algorithm the MessageDigest algorithm
-     * @param ext the file extension.
-     * @param type the checksum type.
+     * @param type a unique identifier for the type
+     * @param ext the list of file extensions
      */
     private ChecksumAlgorithm( String algorithm, String type, String... ext )
     {
@@ -98,20 +102,40 @@ public enum ChecksumAlgorithm {
 
     }
 
+    /**
+     * Returns the message digest algorithm identifier
+     * @return
+     */
     public String getAlgorithm()
     {
         return algorithm;
     }
 
+    /**
+     * Returns the list of extensions
+     * @return
+     */
     public List<String> getExt()
     {
         return ext;
     }
 
+    /**
+     * Returns the checksum identifier
+     * @return
+     */
     public String getType()
     {
         return type;
     }
+
+    /**
+     * Returns the default extension of the current algorithm
+     * @return
+     */
+    public String getDefaultExtension() {
+        return ext.get(0);
+    }
     
     
 }

archiva-modules/archiva-base/archiva-checksum/src/main/java/org/apache/archiva/checksum/ChecksumUtil.java

@@ -0,0 +1,100 @@
+package org.apache.archiva.checksum;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * Utility class that handles multiple checksums for a single file.
+ */
+public class ChecksumUtil {
+
+
+    static final int BUFFER_SIZE = 32768;
+
+    public static void update(List<Checksum> checksumList, Path file ) throws IOException {
+        long fileSize;
+        try (FileChannel channel = FileChannel.open(file, StandardOpenOption.READ )) {
+            fileSize = channel.size();
+            long pos = 0;
+            while (pos < fileSize) {
+                long bufferSize = Math.min(BUFFER_SIZE, fileSize - pos);
+                MappedByteBuffer buffer = channel.map(FileChannel.MapMode.READ_ONLY, pos, bufferSize);
+                for (Checksum checksum : checksumList) {
+                    checksum.update(buffer);
+                    buffer.rewind();
+                }
+                fileSize = channel.size();
+                pos += BUFFER_SIZE;
+            }
+            for (Checksum checksum : checksumList) {
+                checksum.finish();
+            }
+        }
+    }
+
+    public static void update(Checksum checksum, Path file)
+        throws IOException
+    {
+        long fileSize;
+        try (FileChannel channel = FileChannel.open(file, StandardOpenOption.READ )) {
+            fileSize = channel.size();
+            long pos = 0;
+            while (pos<fileSize)
+            {
+                long bufferSize = Math.min(BUFFER_SIZE, fileSize-pos);
+                MappedByteBuffer buffer = channel.map( FileChannel.MapMode.READ_ONLY, pos, bufferSize);
+                checksum.update( buffer );
+                buffer.rewind();
+                fileSize = channel.size();
+                pos += BUFFER_SIZE;
+            }
+            checksum.finish();
+        }
+    }
+
+    public static List<Checksum> initializeChecksums(Path file, List<ChecksumAlgorithm> checksumAlgorithms) throws IOException {
+        final List<Checksum> checksums = newChecksums(checksumAlgorithms);
+        update(checksums, file);
+        return checksums;
+    }
+
+    /**
+     * Returns the list of configured checksum types.
+     *
+     * @param checksumTypes The list of checksum strings
+     * @return The list of checksum objects
+     */
+    public static List<ChecksumAlgorithm> getAlgorithms(List<String> checksumTypes) {
+        return checksumTypes.stream().map(ca ->
+                ChecksumAlgorithm.valueOf(ca.toUpperCase())).collect(Collectors.toList());
+    }
+
+    public static List<Checksum> newChecksums(List<ChecksumAlgorithm> checksumAlgorithms) {
+        return checksumAlgorithms.stream().map( a -> new Checksum(a)).collect(Collectors.toList());
+    }
+}

archiva-modules/archiva-base/archiva-checksum/src/main/java/org/apache/archiva/checksum/ChecksummedFile.java

@@ -23,7 +23,7 @@ import org.apache.archiva.common.utils.FileUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import javax.xml.bind.ValidationException;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.nio.charset.Charset;
 import java.nio.file.Files;
@@ -98,21 +98,21 @@ public class ChecksummedFile
     {
 
         Checksum checksum = new Checksum( checksumAlgorithm );
-        checksum.update( referenceFile );
+        ChecksumUtil.update(checksum, referenceFile );
         return checksum.getChecksum( );
     }
 
     /**
-     * Creates a checksum file of the provided referenceFile.
+     * Writes a checksum file for the referenceFile.
      *
      * @param checksumAlgorithm the hash to use.
      * @return the checksum File that was created.
      * @throws IOException if there was a problem either reading the referenceFile, or writing the checksum file.
      */
-    public Path createChecksum( ChecksumAlgorithm checksumAlgorithm )
+    public Path writeFile(ChecksumAlgorithm checksumAlgorithm )
         throws IOException
     {
-        Path checksumFile = referenceFile.resolveSibling( referenceFile.getFileName( ) + "." + checksumAlgorithm.getExt( ).get( 0 ) );
+        Path checksumFile = referenceFile.resolveSibling( referenceFile.getFileName( ) + "." + checksumAlgorithm.getDefaultExtension() );
         Files.deleteIfExists( checksumFile );
         String checksum = calculateChecksum( checksumAlgorithm );
         Files.write( checksumFile, //
@@ -123,6 +123,8 @@ public class ChecksummedFile
 
     /**
      * Get the checksum file for the reference file and hash.
+     * It returns a file for the given checksum, if one exists with one of the possible extensions.
+     * If it does not exist, a default path will be returned.
      *
      * @param checksumAlgorithm the hash that we are interested in.
      * @return the checksum file to return
@@ -137,7 +139,7 @@ public class ChecksummedFile
                 return file;
             }
         }
-        return referenceFile.resolveSibling( referenceFile.getFileName( ) + "." + checksumAlgorithm.getExt( ).get( 0 ) );
+        return referenceFile.resolveSibling( referenceFile.getFileName( ) + "." + checksumAlgorithm.getDefaultExtension() );
     }
 
     /**
@@ -175,66 +177,65 @@ public class ChecksummedFile
     }
 
     /**
-     * Checks if the checksums are valid for the referenced file.
+     * Checks if the checksum files are valid for the referenced file.
+     * It tries to find a checksum file for each algorithm in the same directory as the referenceFile.
+     * The method returns true, if at least one checksum file exists for one of the given algorithms
+     * and all existing checksum files are valid.
+     *
      * This method throws only exceptions, if throwExceptions is true. Otherwise false will be returned instead.
+     *
+     * It verifies only the existing checksum files. If the checksum file for a particular algorithm does not exist,
+     * but others exist and are valid, it will return true.
+     *
      * @param algorithms The algorithms to verify
      * @param throwExceptions If true, exceptions will be thrown, otherwise false will be returned, if a exception occurred.
-     * @return True, if it is valid, otherwise false.
+     * @return True, if it is valid for all existing checksum files, otherwise false.
      * @throws ChecksumValidationException
      */
     public boolean isValidChecksums( List<ChecksumAlgorithm> algorithms, boolean throwExceptions) throws ChecksumValidationException
     {
 
-        List<Checksum> checksums = new ArrayList<>( algorithms.size() );
-        // Create checksum object for each algorithm.
-        for ( ChecksumAlgorithm checksumAlgorithm : algorithms )
-        {
-            Path checksumFile = getChecksumFile( checksumAlgorithm );
-
-            // Only add algorithm if checksum file exists.
-            if ( Files.exists( checksumFile ) )
-            {
-                checksums.add( new Checksum( checksumAlgorithm ) );
-            }
-        }
-
-        // Any checksums?
-        if ( checksums.isEmpty( ) )
-        {
-            // No checksum objects, no checksum files, default to is invalid.
-            return false;
-        }
-
+        List<Checksum> checksums;
         // Parse file once, for all checksums.
         try
         {
-            Checksum.update( checksums, referenceFile );
+            checksums = ChecksumUtil.initializeChecksums( referenceFile, algorithms );
         }
-        catch ( ChecksumValidationException e )
+        catch (IOException e )
         {
             log.warn( "Unable to update checksum:{}", e.getMessage( ) );
             if (throwExceptions) {
-                throw e;
+                if (e instanceof FileNotFoundException) {
+                    throw new ChecksumValidationException(ChecksumValidationException.ValidationError.FILE_NOT_FOUND, e);
+                } else {
+                    throw new ChecksumValidationException(ChecksumValidationException.ValidationError.READ_ERROR, e);
+                }
             } else {
                 return false;
             }
         }
 
         boolean valid = true;
+        boolean fileExists = false;
+
+        // No file exists -> return false
+        // if at least one file exists:
+        // -> all existing files must be valid
 
         // check the checksum files
         try
         {
+
             for ( Checksum checksum : checksums )
             {
                 ChecksumAlgorithm checksumAlgorithm = checksum.getAlgorithm( );
                 Path checksumFile = getChecksumFile( checksumAlgorithm );
 
-                String expectedChecksum = parseChecksum( checksumFile, checksumAlgorithm, referenceFile.getFileName( ).toString( ), FILE_ENCODING );
+                if (Files.exists(checksumFile)) {
+                    fileExists = true;
+                    String expectedChecksum = parseChecksum(checksumFile, checksumAlgorithm, referenceFile.getFileName().toString(), FILE_ENCODING);
 
-                if ( !checksum.compare( expectedChecksum ) )
-                {
-                    valid = false;
+                    valid &= checksum.compare(expectedChecksum);
                 }
             }
         }
@@ -249,7 +250,7 @@ public class ChecksummedFile
             }
         }
 
-        return valid;
+        return fileExists && valid;
     }
 
     public Path getReferenceFile( )
@@ -259,40 +260,39 @@ public class ChecksummedFile
 
 
 
-    public boolean fixChecksum(ChecksumAlgorithm algorithm) {
+    public UpdateStatusList fixChecksum(ChecksumAlgorithm algorithm) {
         return fixChecksums( Arrays.asList(algorithm) );
     }
+
     /**
-     * Fix or create checksum files for the reference file.
+     * Writes a checksum file, if it does not exist or if it exists and has a different
+     * checksum value.
      *
      * @param algorithms the hashes to check for.
      * @return true if checksums were created successfully.
      */
-    public boolean fixChecksums( List<ChecksumAlgorithm> algorithms )
+    public UpdateStatusList fixChecksums( List<ChecksumAlgorithm> algorithms )
     {
-        List<Checksum> checksums = new ArrayList<>( algorithms.size() );
-        // Create checksum object for each algorithm.
-        for ( ChecksumAlgorithm checksumAlgorithm : algorithms )
-        {
-            checksums.add( new Checksum( checksumAlgorithm ) );
-        }
+        UpdateStatusList result = UpdateStatusList.INITIALIZE(algorithms);
+        List<Checksum> checksums;
 
-        // Any checksums?
-        if ( checksums.isEmpty( ) )
-        {
-            // No checksum objects, no checksum files, default to is valid.
-            return true;
-        }
 
         try
         {
             // Parse file once, for all checksums.
-            Checksum.update( checksums, referenceFile );
+            checksums = ChecksumUtil.initializeChecksums(getReferenceFile(), algorithms);
         }
-        catch ( ChecksumValidationException e )
+        catch (IOException e )
         {
             log.warn( e.getMessage( ), e );
-            return false;
+            result.setTotalError(e);
+            return result;
+        }
+        // Any checksums?
+        if ( checksums.isEmpty( ) )
+        {
+            // No checksum objects, no checksum files, default to is valid.
+            return result;
         }
 
         boolean valid = true;
@@ -316,23 +316,25 @@ public class ChecksummedFile
 
                     if ( !checksum.compare( expectedChecksum ) )
                     {
-                        // create checksum (again)
+                        // overwrite checksum file
                         writeChecksumFile( checksumFile, FILE_ENCODING, checksum.getChecksum( ) );
+                        result.setStatus(checksumAlgorithm,UpdateStatus.UPDATED);
                     }
                 }
                 else
                 {
                     writeChecksumFile( checksumFile, FILE_ENCODING, checksum.getChecksum( ) );
+                    result.setStatus(checksumAlgorithm, UpdateStatus.CREATED);
                 }
             }
             catch ( ChecksumValidationException e )
             {
                 log.warn( e.getMessage( ), e );
-                valid = false;
+                result.setErrorStatus(checksumAlgorithm, e);
             }
         }
 
-        return valid;
+        return result;
 
     }
 
@@ -362,33 +364,33 @@ public class ChecksummedFile
      * </p>
      *
      * @param checksumFile The file where the checksum is stored
-     * @param expectedHash The checksum algorithm to check
-     * @param expectedPath The filename of the reference file
+     * @param checksumAlgorithm The checksum algorithm to check
+     * @param fileName The filename of the reference file
      * @return
      * @throws IOException
      */
-    public String parseChecksum( Path checksumFile, ChecksumAlgorithm expectedHash, String expectedPath, Charset encoding )
+    public String parseChecksum( Path checksumFile, ChecksumAlgorithm checksumAlgorithm, String fileName, Charset encoding )
         throws ChecksumValidationException
     {
-        ChecksumFileContent fc = parseChecksumFile( checksumFile, expectedHash, encoding );
-        if ( fc.isFormatMatch() && !isValidChecksumPattern( fc.getFileReference( ), expectedPath ) )
+        ChecksumFileContent fc = parseChecksumFile( checksumFile, checksumAlgorithm, encoding );
+        if ( fc.isFormatMatch() && !isValidChecksumPattern( fc.getFileReference( ), fileName ) )
         {
             throw new ChecksumValidationException(BAD_CHECKSUM_FILE_REF,
-                "The file reference '" + fc.getFileReference( ) + "' in the checksum file does not match expected file: '" + expectedPath + "'" );
+                "The file reference '" + fc.getFileReference( ) + "' in the checksum file does not match expected file: '" + fileName + "'" );
         } else if (!fc.isFormatMatch()) {
             throw new ChecksumValidationException( BAD_CHECKSUM_FILE, "The checksum file content could not be parsed: "+checksumFile );
         }
         return fc.getChecksum( );
-    }
 
-    public ChecksumFileContent parseChecksumFile( Path checksumFile, ChecksumAlgorithm expectedHash, Charset encoding )
+    }
+    public ChecksumFileContent parseChecksumFile( Path checksumFile, ChecksumAlgorithm checksumAlgorithm, Charset encoding )
     {
         ChecksumFileContent fc = new ChecksumFileContent( );
         String rawChecksumString = FileUtils.readFileToString( checksumFile, encoding );
         String trimmedChecksum = rawChecksumString.replace( '\n', ' ' ).trim( );
 
         // Free-BSD / openssl
-        String regex = expectedHash.getType( ) + "\\s*\\(([^)]*)\\)\\s*=\\s*([a-fA-F0-9]+)";
+        String regex = checksumAlgorithm.getType( ) + "\\s*\\(([^)]*)\\)\\s*=\\s*([a-fA-F0-9]+)";
         Matcher m = Pattern.compile( regex ).matcher( trimmedChecksum );
         if ( m.matches( ) )
         {

archiva-modules/archiva-base/archiva-checksum/src/main/java/org/apache/archiva/checksum/UpdateStatus.java

@@ -0,0 +1,89 @@
+package org.apache.archiva.checksum;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Status of checksum update for specific algorithm.
+ */
+public class UpdateStatus {
+
+    /**
+     * Checksum file did not exist before and was created
+     */
+    public static final int CREATED = 1;
+    /**
+     * Checksum file existed, but content differed
+     */
+    public static final int UPDATED = 2;
+    /**
+     * Nothing changed
+     */
+    public static final int NONE = 0;
+    /**
+     * Error occured during update/creation of the checksum file
+     */
+    public static final int ERROR = -1;
+
+    private final ChecksumAlgorithm algorithm;
+    private final int status;
+    private final Throwable error;
+
+    public UpdateStatus(ChecksumAlgorithm algorithm) {
+        this.algorithm = algorithm;
+        status = NONE;
+        error = null;
+    }
+
+    public UpdateStatus(ChecksumAlgorithm algorithm, int status) {
+        this.algorithm = algorithm;
+        this.status = status;
+        error = null;
+    }
+
+    public UpdateStatus(ChecksumAlgorithm algorithm, Throwable error) {
+        this.algorithm = algorithm;
+        this.status = ERROR;
+        this.error = error;
+    }
+
+    /**
+     * Return the status value.
+     * @return The value
+     */
+    public int getValue() {
+        return status;
+    }
+
+    /**
+     * Return error, if exists, otherwise <code>null</code> will be returned.
+     * @return
+     */
+    public Throwable getError() {
+        return error;
+    }
+
+    /**
+     * Return the algorithm, this status is assigned to.
+     * @return The checksum algorithm
+     */
+    public ChecksumAlgorithm getAlgorithm() {
+        return algorithm;
+    }
+}

archiva-modules/archiva-base/archiva-checksum/src/main/java/org/apache/archiva/checksum/UpdateStatusList.java

@@ -0,0 +1,87 @@
+package org.apache.archiva.checksum;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+/**
+ * Container for a list of update status objects.
+ *
+ * If there is a overall error that is not specific to a algorithm, the total status
+ * flag is set to error.
+ */
+public class UpdateStatusList {
+
+    private int totalStatus = UpdateStatus.NONE;
+    private Throwable error;
+    private Map<ChecksumAlgorithm, UpdateStatus> statusList = new TreeMap<>();
+
+    public UpdateStatusList() {
+
+    }
+
+    public void addStatus(UpdateStatus status) {
+        statusList.put(status.getAlgorithm(), status);
+    }
+
+    public static UpdateStatusList INITIALIZE(List<ChecksumAlgorithm> algorithms) {
+        final UpdateStatusList list = new UpdateStatusList();
+        for(ChecksumAlgorithm algorithm : algorithms) {
+            list.addStatus(new UpdateStatus(algorithm));
+        }
+        return list;
+    }
+
+    public int getTotalStatus() {
+        return totalStatus;
+    }
+
+    public void setTotalError(Throwable e) {
+        this.error = e;
+        this.totalStatus = UpdateStatus.ERROR;
+    }
+
+    public Throwable getTotalError() {
+        return error;
+    }
+
+    public List<UpdateStatus> getStatusList() {
+        return new ArrayList(statusList.values());
+    }
+
+    public void setStatus(ChecksumAlgorithm algorithm, UpdateStatus status) {
+        statusList.put(algorithm, status);
+    }
+
+    public void setStatus(ChecksumAlgorithm algorithm, int status) {
+        statusList.put(algorithm, new UpdateStatus(algorithm, status));
+    }
+
+    public void setErrorStatus(ChecksumAlgorithm algorithm, Throwable e) {
+        statusList.put(algorithm, new UpdateStatus(algorithm,e));
+    }
+
+    public UpdateStatus getStatus(ChecksumAlgorithm algorithm) {
+        return statusList.get(algorithm);
+    }
+}

archiva-modules/archiva-base/archiva-checksum/src/test/java/org/apache/archiva/checksum/ChecksumTest.java

@@ -19,15 +19,12 @@ package org.apache.archiva.checksum;
  * under the License.
  */
 
-import junit.framework.TestCase;
 import org.apache.archiva.common.utils.FileUtils;
 import org.apache.archiva.test.utils.ArchivaBlockJUnit4ClassRunner;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 
-import java.io.ByteArrayInputStream;
 import java.io.IOException;
-import java.nio.ByteBuffer;
 import java.nio.charset.Charset;
 import java.nio.file.Path;
 import java.util.ArrayList;
@@ -85,7 +82,7 @@ public class ChecksumTest
         FileUtils.writeStringToFile( checkFile, FILE_ENCODING,  "You know, I'm sick of following my dreams, man. "
             + "I'm just going to ask where they're going and hook up with 'em later. - Mitch Hedberg");
 
-        Checksum.update( checksums, checkFile );
+        ChecksumUtil.update( checksums, checkFile );
 
         assertEquals( "Checksum SHA1", "e396119ae0542e85a74759602fd2f81e5d36d762", checksumSha1.getChecksum() );
         assertEquals( "Checksum MD5", "21c2c5ca87ec018adacb2e2fb3432219", checksumMd5.getChecksum() );

archiva-modules/archiva-base/archiva-checksum/src/test/java/org/apache/archiva/checksum/ChecksummedFileTest.java

@@ -130,7 +130,7 @@ public class ChecksummedFileTest
     {
         Path testableJar = createTestableJar( "examples/redback-authz-open.jar" );
         ChecksummedFile checksummedFile = new ChecksummedFile( testableJar );
-        checksummedFile.createChecksum( ChecksumAlgorithm.SHA1 );
+        checksummedFile.writeFile( ChecksumAlgorithm.SHA1 );
         Path hashFile = checksummedFile.getChecksumFile( ChecksumAlgorithm.SHA1 );
         assertTrue( "ChecksumAlgorithm file should exist.", Files.exists(hashFile) );
         String hashContents = org.apache.commons.io.FileUtils.readFileToString( hashFile.toFile(), "UTF-8" );
@@ -152,8 +152,10 @@ public class ChecksummedFileTest
         assertFalse( "ChecksummedFile.isValid(SHA1) == false",
                      checksummedFile.isValidChecksum( ChecksumAlgorithm.SHA1 ) );
 
-        boolean fixed = checksummedFile.fixChecksums( Arrays.asList( ChecksumAlgorithm.SHA1 ) );
-        assertTrue( "ChecksummedFile.fixChecksums() == true", fixed );
+        UpdateStatusList fixed = checksummedFile.fixChecksums( Arrays.asList( ChecksumAlgorithm.SHA1 ) );
+        assertEquals(1, fixed.getStatusList().size());
+        assertFalse(fixed.getTotalStatus()==UpdateStatus.ERROR);
+        assertTrue( "ChecksummedFile.fixChecksums() == true", fixed.getStatusList().get(0).getValue()==UpdateStatus.UPDATED );
 
         assertTrue( "ChecksummedFile.isValid(SHA1) == true",
                     checksummedFile.isValidChecksum( ChecksumAlgorithm.SHA1 ) );

archiva-modules/archiva-base/archiva-configuration/src/main/java/org/apache/archiva/configuration/ArchivaRuntimeConfiguration.java

@@ -19,6 +19,10 @@ package org.apache.archiva.configuration;
  * under the License.
  */
 
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
 /**
  * 
  *         The runtime configuration.
@@ -76,6 +80,11 @@ public class ArchivaRuntimeConfiguration
      */
     private String languageRange = "en,fr,de";
 
+    /**
+     * List of checksum types (algorithms) that should be applied to repository artifacts.
+     */
+    private List<String> checksumTypes = new ArrayList(Arrays.asList("MD5","SHA1","SHA256"));
+
 
       //-----------/
      //- Methods -/
@@ -233,4 +242,49 @@ public class ArchivaRuntimeConfiguration
         this.urlFailureCacheConfiguration = urlFailureCacheConfiguration;
     } //-- void setUrlFailureCacheConfiguration( CacheConfiguration )
 
+
+    /**
+     * Returns the list of checksum types to generate
+     * @return
+     */
+    public List<String> getChecksumTypes()
+    {
+        if ( this.checksumTypes == null )
+        {
+            this.checksumTypes = new java.util.ArrayList<String>();
+        }
+
+        return this.checksumTypes;
+    }
+
+    /**
+     * Adds a checksum type
+     * @param type
+     */
+    public void addChecksumType(String type) {
+
+        if (!getChecksumTypes().contains(type)) {
+            getChecksumTypes().add(type);
+        }
+    }
+
+    /**
+     * Removes a checksum type
+     * @param type
+     */
+    public void removeChecksumType(String type) {
+        getChecksumTypes().remove(type);
+    }
+
+    /**
+     * Set all checksum types
+     * @param checksumTypes
+     */
+    public void setChecksumTypes(List<String> checksumTypes) {
+        if (checksumTypes!=null) {
+            getChecksumTypes().clear();
+            getChecksumTypes().addAll(checksumTypes);
+        }
+    }
+
 }

archiva-modules/archiva-base/archiva-configuration/src/main/java/org/apache/archiva/configuration/io/registry/ConfigurationRegistryReader.java

@@ -1423,6 +1423,9 @@ public class ConfigurationRegistryReader {
 
         value.setLanguageRange(languageRange);
 
+        List<String> checksumTypeList = registry.getList(prefix + "checksumTypes.type");
+        value.setChecksumTypes(checksumTypeList);
+
         return value;
     }
 

archiva-modules/archiva-base/archiva-configuration/src/main/java/org/apache/archiva/configuration/io/registry/ConfigurationRegistryWriter.java

@@ -24,6 +24,7 @@ import org.apache.archiva.configuration.*;
 import org.apache.archiva.redback.components.registry.Registry;
 
 import java.util.Iterator;
+import java.util.List;
 
 // Util imports
 // Model class imports
@@ -37,6 +38,20 @@ public class ConfigurationRegistryWriter {
         writeConfiguration("", model, registry);
     }
 
+    private void writeList(Registry registry, List<String> subList, String subsetPath, String elementName) {
+        if (subList != null && subList.size() > 0
+        ) {
+            registry.removeSubset(subsetPath);
+
+            int count = 0;
+            for (Iterator<String> iter = subList.iterator(); iter.hasNext(); count++) {
+                String name = subsetPath + "." + elementName + "(" + count + ")";
+                String value = iter.next();
+                registry.setString(name, value);
+            }
+        }
+    }
+
     private void writeConfiguration(String prefix, Configuration value, Registry registry) {
         if (value != null) {
             if (value.getVersion() != null
@@ -882,6 +897,7 @@ public class ConfigurationRegistryWriter {
                 String languageRange = "languageRange";
                 registry.setString(prefix + languageRange, value.getLanguageRange());
             }
+            writeList(registry, value.getChecksumTypes(), prefix+"checksumTypes", "type");
         }
     }
 

archiva-modules/archiva-base/archiva-configuration/src/main/resources/org/apache/archiva/configuration/default-archiva.xml

@@ -142,6 +142,14 @@
     </ui>
   </webapp>
 
+  <archivaRuntimeConfiguration>
+    <checksumTypes>
+      <type>MD5</type>
+      <type>SHA1</type>
+      <type>SHA256</type>
+    </checksumTypes>
+  </archivaRuntimeConfiguration>
+
   <redbackRuntimeConfiguration>
     <userManagerImpls>
       <userManagerImpl>jpa</userManagerImpl>

archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/archiva/consumers/core/ArtifactMissingChecksumsConsumer.java

@@ -19,8 +19,7 @@ package org.apache.archiva.consumers.core;
  * under the License.
  */
 
-import org.apache.archiva.checksum.ChecksumAlgorithm;
-import org.apache.archiva.checksum.ChecksummedFile;
+import org.apache.archiva.checksum.*;
 import org.apache.archiva.configuration.ArchivaConfiguration;
 import org.apache.archiva.configuration.FileTypes;
 import org.apache.archiva.consumers.AbstractMonitoredConsumer;
@@ -58,14 +57,12 @@ public class ArtifactMissingChecksumsConsumer
 
     private String id = "create-missing-checksums";
 
-    private String description = "Create Missing and/or Fix Invalid Checksums (.sha1, .md5)";
+    private String description = "Create Missing and/or Fix Invalid Checksum files.";
 
     private ArchivaConfiguration configuration;
 
     private FileTypes filetypes;
 
-    private ChecksummedFile checksum;
-
     private static final String TYPE_CHECKSUM_NOT_FILE = "checksum-bad-not-file";
 
     private static final String TYPE_CHECKSUM_CANNOT_CALC = "checksum-calc-failure";
@@ -75,6 +72,7 @@ public class ArtifactMissingChecksumsConsumer
     private Path repositoryDir;
 
     private List<String> includes = new ArrayList<>( 0 );
+    private List<ChecksumAlgorithm> algorithms;
 
     @Inject
     public ArtifactMissingChecksumsConsumer( ArchivaConfiguration configuration, FileTypes filetypes )
@@ -141,8 +139,36 @@ public class ArtifactMissingChecksumsConsumer
     public void processFile( String path )
         throws ConsumerException
     {
-        createFixChecksum( path, ChecksumAlgorithm.SHA1 );
-        createFixChecksum( path, ChecksumAlgorithm.MD5 );
+        Path artifactPath = repositoryDir.resolve(path);
+        ChecksummedFile csFile = new ChecksummedFile(artifactPath);
+        UpdateStatusList result = csFile.fixChecksums(algorithms);
+        if (result.getTotalStatus()== UpdateStatus.ERROR) {
+            log.warn( "Error accessing file {}. ", path );
+            triggerConsumerWarning( TYPE_CHECKSUM_NOT_FILE,
+                    "Error accessing file " + path + "." );
+        } else {
+            result.getStatusList().stream().forEach(st ->
+                    triggerInfo(path, st));
+        }
+    }
+
+    private void triggerInfo(String path, UpdateStatus status) {
+        switch (status.getValue()) {
+            case UpdateStatus.ERROR:
+                log.error( "Cannot create checksum for file {} :", path, status.getError() );
+                triggerConsumerError( TYPE_CHECKSUM_CANNOT_CREATE, "Cannot create checksum for file " + path +
+                        ": " + status.getError().getMessage( ) );
+                break;
+            case UpdateStatus.CREATED:
+                log.info( "Created missing checksum file {}", path );
+                triggerConsumerInfo( "Created missing checksum file " + path );
+                break;
+            case UpdateStatus.UPDATED:
+                log.info( "Fixed checksum file {}", path );
+                triggerConsumerInfo( "Fixed checksum file " + path );
+                break;
+
+        }
     }
 
     @Override
@@ -152,44 +178,6 @@ public class ArtifactMissingChecksumsConsumer
         processFile( path );
     }
 
-    private void createFixChecksum( String path, ChecksumAlgorithm checksumAlgorithm )
-    {
-        Path artifactFile = repositoryDir.resolve(path);
-        Path checksumFile = repositoryDir.resolve(path + "." + checksumAlgorithm.getExt( ).get(0) );
-
-        if ( Files.exists(checksumFile) )
-        {
-            checksum = new ChecksummedFile( artifactFile);
-            if ( !checksum.isValidChecksum( checksumAlgorithm ) )
-            {
-                checksum.fixChecksum( checksumAlgorithm );
-                log.info( "Fixed checksum file {}", checksumFile.toAbsolutePath( ) );
-                triggerConsumerInfo( "Fixed checksum file " + checksumFile.toAbsolutePath( ) );
-            }
-        }
-        else if ( !Files.exists(checksumFile) )
-        {
-            checksum = new ChecksummedFile( artifactFile);
-            try
-            {
-                checksum.createChecksum( checksumAlgorithm );
-                log.info( "Created missing checksum file {}", checksumFile.toAbsolutePath( ) );
-                triggerConsumerInfo( "Created missing checksum file " + checksumFile.toAbsolutePath( ) );
-            }
-            catch ( IOException e )
-            {
-                log.error( "Cannot create checksum for file {} :", checksumFile, e );
-                triggerConsumerError( TYPE_CHECKSUM_CANNOT_CREATE, "Cannot create checksum for file " + checksumFile +
-                    ": " + e.getMessage( ) );
-            }
-        }
-        else
-        {
-            log.warn( "Checksum file {} is not a file. ", checksumFile.toAbsolutePath( ) );
-            triggerConsumerWarning( TYPE_CHECKSUM_NOT_FILE,
-                "Checksum file " + checksumFile.toAbsolutePath( ) + " is not a file." );
-        }
-    }
 
     /*
     @Override
@@ -222,5 +210,6 @@ public class ArtifactMissingChecksumsConsumer
         //configuration.addChangeListener( this );
 
         initIncludes( );
+        algorithms = ChecksumUtil.getAlgorithms(configuration.getConfiguration().getArchivaRuntimeConfiguration().getChecksumTypes());
     }
 }

archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/main/java/org/apache/archiva/consumers/core/ValidateChecksumConsumer.java

@@ -159,7 +159,7 @@ public class ValidateChecksumConsumer
     @PostConstruct
     public void initialize( )
     {
-        Set<String> extensions = ChecksumAlgorithm.getExtensions();
+        Set<String> extensions = ChecksumAlgorithm.getAllExtensions();
         includes = new ArrayList<>( extensions.size() );
         for ( String ext : extensions )
         {

archiva-modules/archiva-base/archiva-consumers/archiva-core-consumers/src/test/java/org/apache/archiva/consumers/core/AbstractArtifactConsumerTest.java

@@ -63,6 +63,9 @@ public abstract class AbstractArtifactConsumerTest
             (FileType) archivaConfiguration.getConfiguration().getRepositoryScanning().getFileTypes().get( 0 );
         assertEquals( FileTypes.ARTIFACTS, fileType.getId() );
         fileType.addPattern( "**/*.xml" );
+        archivaConfiguration.getConfiguration().getArchivaRuntimeConfiguration().addChecksumType("MD5");
+        archivaConfiguration.getConfiguration().getArchivaRuntimeConfiguration().addChecksumType("SHA1");
+        archivaConfiguration.getConfiguration().getArchivaRuntimeConfiguration().addChecksumType("SHA256");
 
         repoLocation = Paths.get( "target/test-" + getName() + "/test-repo" );
     }

archiva-modules/archiva-base/archiva-policies/src/main/java/org/apache/archiva/policies/ChecksumPolicy.java

@@ -21,6 +21,7 @@ package org.apache.archiva.policies;
 
 import org.apache.archiva.checksum.ChecksumAlgorithm;
 import org.apache.archiva.checksum.ChecksummedFile;
+import org.apache.archiva.checksum.UpdateStatus;
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -146,7 +147,7 @@ public class ChecksumPolicy
         if ( FIX.equals( policySetting ) )
         {
             ChecksummedFile checksum = new ChecksummedFile( localFile );
-            if ( checksum.fixChecksums( algorithms ) )
+            if ( checksum.fixChecksums( algorithms ).getTotalStatus() != UpdateStatus.ERROR )
             {
                 log.debug( "Checksum policy set to FIX, checksum files have been updated." );
                 return;

archiva-modules/archiva-web/archiva-web-common/src/main/java/org/apache/archiva/web/api/DefaultFileUploadService.java

@@ -25,9 +25,11 @@ import org.apache.archiva.admin.model.admin.ArchivaAdministration;
 import org.apache.archiva.admin.model.beans.ManagedRepository;
 import org.apache.archiva.admin.model.managed.ManagedRepositoryAdmin;
 import org.apache.archiva.checksum.ChecksumAlgorithm;
+import org.apache.archiva.checksum.ChecksumUtil;
 import org.apache.archiva.checksum.ChecksummedFile;
 import org.apache.archiva.common.utils.VersionComparator;
 import org.apache.archiva.common.utils.VersionUtil;
+import org.apache.archiva.configuration.ArchivaConfiguration;
 import org.apache.archiva.maven2.metadata.MavenMetadataReader;
 import org.apache.archiva.metadata.model.facets.AuditEvent;
 import org.apache.archiva.model.ArchivaRepositoryMetadata;
@@ -59,6 +61,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Service;
 
+import javax.annotation.PostConstruct;
 import javax.inject.Inject;
 import javax.inject.Named;
 import javax.servlet.http.HttpServletRequest;
@@ -74,9 +77,14 @@ import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.*;
 import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.stream.Collectors;
 
 /**
+ *
+ * Service for uploading files to the repository.
+ *
  * @author Olivier Lamy
+ * @author Martin Stockhammer
  */
 @Service("fileUploadService#rest")
 public class DefaultFileUploadService
@@ -96,7 +104,10 @@ public class DefaultFileUploadService
     @Inject
     private ArchivaAdministration archivaAdministration;
 
-    private List<ChecksumAlgorithm> algorithms = Arrays.asList(ChecksumAlgorithm.SHA1, ChecksumAlgorithm.MD5);
+    @Inject
+    ArchivaConfiguration configuration;
+
+    private List<ChecksumAlgorithm> algorithms;
 
     private final String FS = FileSystems.getDefault().getSeparator();
 
@@ -111,6 +122,11 @@ public class DefaultFileUploadService
                 StringUtils.trim(URLDecoder.decode(IOUtils.toString(attachment.getDataHandler().getInputStream(), "UTF-8"), "UTF-8"));
     }
 
+    @PostConstruct
+    private void initialize() {
+        algorithms = ChecksumUtil.getAlgorithms(configuration.getConfiguration().getArchivaRuntimeConfiguration().getChecksumTypes());
+    }
+
     @Override
     public FileMetadata post(MultipartBody multipartBody)
             throws ArchivaRestServiceException {
@@ -233,12 +249,10 @@ public class DefaultFileUploadService
     @Override
     public List<FileMetadata> getSessionFileMetadatas()
             throws ArchivaRestServiceException {
-        @SuppressWarnings("unchecked") List<FileMetadata> fileMetadatas =
-                (List<FileMetadata>) httpServletRequest.getSession().getAttribute(FILES_SESSION_KEY);
-
-        return fileMetadatas == null ? Collections.<FileMetadata>emptyList() : fileMetadatas;
+        return getSessionFilesList();
     }
 
+
     private boolean hasValidChars(String checkString) {
         if (checkString.contains(FS)) {
             return false;
@@ -427,8 +441,9 @@ public class DefaultFileUploadService
                 filename = filename.replaceAll(VersionUtil.SNAPSHOT, timestamp + "-" + newBuildNumber);
             }
 
-            boolean fixChecksums =
-                    !(archivaAdministration.getKnownContentConsumers().contains("create-missing-checksums"));
+            // We always fix checksums for newly uploaded files, even if the content consumer is active.
+            boolean fixChecksums = true;
+            // !(archivaAdministration.getKnownContentConsumers().contains("create-missing-checksums"));
 
             try {
                 Path targetFile = targetPath.resolve(filename);

archiva-modules/archiva-web/archiva-web-common/src/test/java/org/apache/archiva/upload/UploadArtifactsTest.java

@@ -21,6 +21,7 @@ package org.apache.archiva.upload;
 import com.fasterxml.jackson.jaxrs.json.JacksonJaxbJsonProvider;
 import org.apache.archiva.configuration.ArchivaConfiguration;
 import org.apache.archiva.redback.rest.services.AbstractRestServicesTest;
+import org.apache.archiva.redback.rest.services.FakeCreateAdminService;
 import org.apache.archiva.rest.api.services.ArchivaRestServiceException;
 import org.apache.archiva.test.utils.ArchivaBlockJUnit4ClassRunner;
 import org.apache.archiva.web.api.FileUploadService;
@@ -248,4 +249,26 @@ public class UploadArtifactsTest
         service.post(body);
         service.save("internal", "org.apache.archiva", "archiva-model", "1.2", "jar", true);
     }
+
+    @Test
+    public void saveFileWithOtherExtension() throws IOException, ArchivaRestServiceException {
+        log.debug("Starting saveFileWithOtherExtension()");
+
+        Path path = Paths.get("target/appserver-base/repositories/internal/data/repositories/internal/org/apache/archiva/archiva-model/1.2/archiva-model-1.2.bin");
+        log.debug("Jar exists: {}",Files.exists(path));
+        Files.deleteIfExists(path);
+        Path pomPath = Paths.get("target/appserver-base/repositories/internal/data/repositories/internal/org/apache/archiva/archiva-model/1.2/archiva-model-1.2.pom");
+        Files.deleteIfExists(pomPath);
+        FileUploadService service = getUploadService();
+        service.clearUploadedFiles();
+        Path file = Paths.get("src/test/repositories/snapshot-repo/org/apache/archiva/archiva-model/1.4-M4-SNAPSHOT/archiva-model-1.4-M4-20130425.081822-1.jar");
+        log.debug("Upload file exists: {}", Files.exists(file));
+        final Attachment fileAttachment = new AttachmentBuilder().object(Files.newInputStream(file)).contentDisposition(new ContentDisposition("form-data; filename=\"archiva-model.bin\"; name=\"files[]\"")).build();
+        MultipartBody body = new MultipartBody(fileAttachment);
+        service.post(body);
+        assertTrue(service.save("internal", "org.apache.archiva", "archiva-model", "1.2", "bin", false));
+        assertTrue(Files.exists(path));
+    }
+
+
 }