Skip to content

feat: Support pre-calculated checksums in SIP generation #369

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
JohannesKarlsen99 wants to merge 6 commits into
base: master
Choose a base branch
from
Open

feat: Support pre-calculated checksums in SIP generation #369

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
cf93ed7
feat: Support pre-calculated checksums in SIP generation
JohannesKarlsen99 Dec 18, 2025
7fd178a
Update test to comply with checkstyle
JohannesKarlsen99 Dec 18, 2025
b5b016a
Update code to comply with checkstyle
JohannesKarlsen99 Dec 18, 2025
c1cb6fa
Previously, pre-calculated checksums set on IPFile objects were only
JohannesKarlsen99 Jan 13, 2026
0c9b424
update code to comply with checkstyle
JohannesKarlsen99 Jan 13, 2026
157dfde
update code to comply with checkstyle
JohannesKarlsen99 Jan 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions src/main/java/org/roda_project/commons_ip2/model/IPFileInterface.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,38 @@
*/
public interface IPFileInterface extends Serializable {

/**
* Gets the relative folders for this file.
*
* @return list of relative folder names
*/
List<String> getRelativeFolders();

/**
* Gets the file name.
*
* @return the file name
*/
String getFileName();

/**
* Gets the path to the file.
*
* @return the file path
*/
Path getPath();

/**
* Gets the pre-calculated checksum of the file, if available.
*
* @return the checksum value, or null if not set
*/
String getChecksum();

/**
* Gets the algorithm used for the pre-calculated checksum.
*
* @return the checksum algorithm (e.g., "SHA-256"), or null if not set
*/
String getChecksumAlgorithm();
}
18 changes: 18 additions & 0 deletions src/main/java/org/roda_project/commons_ip2/model/IPFileShallow.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,4 +103,22 @@ public String getFileName() {
public Path getPath() {
throw new UnsupportedOperationException("IPFileShallow does not support this method");
}

@Override
public String getChecksum() {
// IPFileShallow may have checksum in fileType if set externally
if (fileType != null && fileType.getCHECKSUM() != null) {
return fileType.getCHECKSUM();
}
return "";
}

@Override
public String getChecksumAlgorithm() {
// IPFileShallow may have checksum algorithm in fileType if set externally
if (fileType != null && fileType.getCHECKSUMTYPE() != null) {
return fileType.getCHECKSUMTYPE();
}
return "";
}
}
38 changes: 24 additions & 14 deletions src/main/java/org/roda_project/commons_ip2/model/impl/eark/EARKUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import org.apache.commons.lang3.StringUtils;
import org.roda_project.commons_ip.model.ParseException;
Expand Down Expand Up @@ -92,7 +90,8 @@ protected void addDescriptiveMetadataToZipAndMETS(Map<String, ZipEntryInfo> zipE
descriptiveFilePath = IPConstants.REPRESENTATIONS_FOLDER + representationId + IPConstants.ZIP_PATH_SEPARATOR
+ descriptiveFilePath;
}
ZIPUtils.addMdRefFileToZip(zipEntries, file.getPath(), descriptiveFilePath, mdRef);
ZIPUtils.addMdRefFileToZip(zipEntries, file.getPath(), descriptiveFilePath, mdRef,
file.getChecksum(), file.getChecksumAlgorithm());
}
}
}
Expand All @@ -114,7 +113,8 @@ protected void addPreservationMetadataToZipAndMETS(Map<String, ZipEntryInfo> zip
preservationMetadataPath = IPConstants.REPRESENTATIONS_FOLDER + representationId
+ IPConstants.ZIP_PATH_SEPARATOR + preservationMetadataPath;
}
ZIPUtils.addMdRefFileToZip(zipEntries, file.getPath(), preservationMetadataPath, mdRef);
ZIPUtils.addMdRefFileToZip(zipEntries, file.getPath(), preservationMetadataPath, mdRef,
file.getChecksum(), file.getChecksumAlgorithm());
}
}
}
Expand All @@ -136,7 +136,8 @@ protected void addOtherMetadataToZipAndMETS(Map<String, ZipEntryInfo> zipEntries
otherMetadataPath = IPConstants.REPRESENTATIONS_FOLDER + representationId + IPConstants.ZIP_PATH_SEPARATOR
+ otherMetadataPath;
}
ZIPUtils.addMdRefFileToZip(zipEntries, file.getPath(), otherMetadataPath, mdRef);
ZIPUtils.addMdRefFileToZip(zipEntries, file.getPath(), otherMetadataPath, mdRef,
file.getChecksum(), file.getChecksumAlgorithm());
}
}
}
Expand All @@ -158,7 +159,8 @@ protected void addTechnicalMetadataToZipAndMETS(Map<String, ZipEntryInfo> zipEnt
technicalMetadataPath = IPConstants.REPRESENTATIONS_FOLDER + representationId + IPConstants.ZIP_PATH_SEPARATOR
+ technicalMetadataPath;
}
ZIPUtils.addMdRefFileToZip(zipEntries, file.getPath(), technicalMetadataPath, mdRef);
ZIPUtils.addMdRefFileToZip(zipEntries, file.getPath(), technicalMetadataPath, mdRef,
file.getChecksum(), file.getChecksumAlgorithm());
}
}
}
Expand All @@ -180,7 +182,8 @@ protected void addSourceMetadataToZipAndMETS(Map<String, ZipEntryInfo> zipEntrie
sourceMetadataPath = IPConstants.REPRESENTATIONS_FOLDER + representationId + IPConstants.ZIP_PATH_SEPARATOR
+ sourceMetadataPath;
}
ZIPUtils.addMdRefFileToZip(zipEntries, file.getPath(), sourceMetadataPath, mdRef);
ZIPUtils.addMdRefFileToZip(zipEntries, file.getPath(), sourceMetadataPath, mdRef,
file.getChecksum(), file.getChecksumAlgorithm());
}
}
}
Expand All @@ -202,7 +205,8 @@ protected void addRightsMetadataToZipAndMETS(Map<String, ZipEntryInfo> zipEntrie
rightsMetadataPath = IPConstants.REPRESENTATIONS_FOLDER + representationId + IPConstants.ZIP_PATH_SEPARATOR
+ rightsMetadataPath;
}
ZIPUtils.addMdRefFileToZip(zipEntries, file.getPath(), rightsMetadataPath, mdRef);
ZIPUtils.addMdRefFileToZip(zipEntries, file.getPath(), rightsMetadataPath, mdRef,
file.getChecksum(), file.getChecksumAlgorithm());
}
}
}
Expand Down Expand Up @@ -357,7 +361,8 @@ private void addRepresentationDataFilesToZipErmsAndMETS(IPInterface ip, Map<Stri
dataFilePath = IPConstants.DATA_FOLDER + dataFilePath;
dataFilePath = IPConstants.REPRESENTATIONS_FOLDER + representationId + IPConstants.ZIP_PATH_SEPARATOR
+ dataFilePath;
ZIPUtils.addFileTypeFileToZip(zipEntries, file.getPath(), dataFilePath, fileType);
ZIPUtils.addFileTypeFileToZip(zipEntries, file.getPath(), dataFilePath, fileType,
file.getChecksum(), file.getChecksumAlgorithm());
} else if (file instanceof IPFileShallow shallow && (shallow.getFileLocation() != null)) {
metsGenerator.addDataFileToMETS(representationMETSWrapper, shallow);
}
Expand Down Expand Up @@ -397,7 +402,8 @@ private void addRepresentationDataFilesToZipSiardAndMETS(IPInterface ip, Map<Str
dataFilePath = IPConstants.DATA_FOLDER + dataFilePath;
dataFilePath = IPConstants.REPRESENTATIONS_FOLDER + representationId + IPConstants.ZIP_PATH_SEPARATOR
+ dataFilePath;
ZIPUtils.addFileTypeFileToZip(zipEntries, file.getPath(), dataFilePath, fileType);
ZIPUtils.addFileTypeFileToZip(zipEntries, file.getPath(), dataFilePath, fileType,
file.getChecksum(), file.getChecksumAlgorithm());
} else if (file instanceof IPFileShallow shallow && (shallow.getFileLocation() != null)) {
metsGenerator.addDataFileToMETS(representationMETSWrapper, shallow);
}
Expand Down Expand Up @@ -433,7 +439,8 @@ protected void addRepresentationDataFilesToZipAndMETS(IPInterface ip, Map<String

dataFilePath = IPConstants.REPRESENTATIONS_FOLDER + representationId + IPConstants.ZIP_PATH_SEPARATOR
+ dataFilePath;
ZIPUtils.addFileTypeFileToZip(zipEntries, file.getPath(), dataFilePath, fileType);
ZIPUtils.addFileTypeFileToZip(zipEntries, file.getPath(), dataFilePath, fileType,
file.getChecksum(), file.getChecksumAlgorithm());
} else if (file instanceof IPFileShallow shallow && (shallow.getFileLocation() != null)) {
metsGenerator.addDataFileToMETS(representationMETSWrapper, shallow);
}
Expand Down Expand Up @@ -465,7 +472,8 @@ protected void addSchemasToZipAndMETS(Map<String, ZipEntryInfo> zipEntries, Mets
schemaFilePath = IPConstants.REPRESENTATIONS_FOLDER + representationId + IPConstants.ZIP_PATH_SEPARATOR
+ schemaFilePath;
}
ZIPUtils.addFileTypeFileToZip(zipEntries, schema.getPath(), schemaFilePath, fileType);
ZIPUtils.addFileTypeFileToZip(zipEntries, schema.getPath(), schemaFilePath, fileType,
schema.getChecksum(), schema.getChecksumAlgorithm());
}
}
}
Expand All @@ -486,7 +494,8 @@ protected void addDocumentationToZipAndMETS(Map<String, ZipEntryInfo> zipEntries
documentationFilePath = IPConstants.REPRESENTATIONS_FOLDER + representationId + IPConstants.ZIP_PATH_SEPARATOR
+ documentationFilePath;
}
ZIPUtils.addFileTypeFileToZip(zipEntries, doc.getPath(), documentationFilePath, fileType);
ZIPUtils.addFileTypeFileToZip(zipEntries, doc.getPath(), documentationFilePath, fileType,
doc.getChecksum(), doc.getChecksumAlgorithm());
}
}
}
Expand Down Expand Up @@ -547,7 +556,8 @@ protected void addSubmissionsToZipAndMETS(final Map<String, ZipEntryInfo> zipEnt
+ ModelUtils.getFoldersFromList(submission.getRelativeFolders()) + submission.getFileName();
final FileType fileType = metsGenerator.addSubmissionFileToMETS(metsWrapper, submissionFilePath,
submission.getPath());
ZIPUtils.addFileTypeFileToZip(zipEntries, submission.getPath(), submissionFilePath, fileType);
ZIPUtils.addFileTypeFileToZip(zipEntries, submission.getPath(), submissionFilePath, fileType,
submission.getChecksum(), submission.getChecksumAlgorithm());
}
}
}
Expand Down
60 changes: 43 additions & 17 deletions ...rg/roda_project/commons_ip2/model/impl/eark/out/writers/strategy/FolderWriteStrategy.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import java.nio.file.Paths;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.HexFormat;
import java.util.Map;

import org.apache.commons.io.IOUtils;
Expand All @@ -19,13 +20,13 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import jakarta.xml.bind.DatatypeConverter;

/**
* @author Miguel Guimarães <mguimaraes@keep.pt>
*/
public class FolderWriteStrategy implements WriteStrategy {
private static final Logger LOGGER = LoggerFactory.getLogger(FolderWriteStrategy.class);
private static final int BUFFER_SIZE = 4096;

Check warning on line 29 in src/main/java/org/roda_project/commons_ip2/model/impl/eark/out/writers/strategy/FolderWriteStrategy.java

View workflow job for this annotation

GitHub Actions / Checkstyle job 

[checkstyle] reported by reviewdog 🐶
Missing a Javadoc comment.

Raw Output:
/github/workspace/./src/main/java/org/roda_project/commons_ip2/model/impl/eark/out/writers/strategy/FolderWriteStrategy.java:29:3: warning: Missing a Javadoc comment. (com.puppycrawl.tools.checkstyle.checks.javadoc.JavadocVariableCheck)
private Path destinationPath;

@Override
Expand Down Expand Up @@ -54,19 +55,23 @@
throw new UnsupportedOperationException("Method not implemented");
}

private void writeToPath(final Map<String, ZipEntryInfo> entries, final Path path, String checksumAlgorithm)
private void writeToPath(final Map<String, ZipEntryInfo> entries, final Path path, final String checksumAlgorithm)
throws IPException, InterruptedException {
try {
Files.createDirectories(path);
for (ZipEntryInfo zipEntryInfo : entries.values()) {
if (Thread.interrupted()) {
throw new InterruptedException();
}
// Save pre-calculated checksum BEFORE it gets overwritten
final String preCalculatedChecksum = zipEntryInfo.getChecksum();
final String preCalculatedAlgorithm = zipEntryInfo.getChecksumAlgorithm();

zipEntryInfo.setChecksum(checksumAlgorithm);
zipEntryInfo.prepareEntryForZipping();
LOGGER.debug("Writing file {}", zipEntryInfo.getFilePath());
final Path outputPath = Paths.get(path.toString(), zipEntryInfo.getName());
writeFileToPath(zipEntryInfo, outputPath, checksumAlgorithm);
writeFileToPath(zipEntryInfo, outputPath, checksumAlgorithm, preCalculatedChecksum, preCalculatedAlgorithm);
}
} catch (final IOException | NoSuchAlgorithmException e) {
LOGGER.debug("Error in write method", e);
Expand All @@ -93,29 +98,50 @@
return path;
}

private void writeFileToPath(final ZipEntryInfo zipEntryInfo, final Path outputPath, String checksumAlgorithm)
private void writeFileToPath(final ZipEntryInfo zipEntryInfo, final Path outputPath, final String checksumAlgorithm,
final String preCalculatedChecksum, final String preCalculatedAlgorithm)
throws IOException, NoSuchAlgorithmException {
InputStream is = null;
OutputStream os = null;
try {

is = Files.newInputStream(zipEntryInfo.getFilePath());

Files.createDirectories(outputPath.getParent());
os = Files.newOutputStream(outputPath);

final byte[] buffer = new byte[4096];
final MessageDigest complete = MessageDigest.getInstance(checksumAlgorithm);
int numRead;
do {
numRead = is.read(buffer);
if (numRead > 0) {
complete.update(buffer, 0, numRead);
os.write(buffer, 0, numRead);
}
} while (numRead != -1);

setChecksum(zipEntryInfo, DatatypeConverter.printHexBinary(complete.digest()), checksumAlgorithm);
// Check if file already has a pre-calculated checksum matching the requested algorithm
final boolean hasValidPreCalculatedChecksum = preCalculatedChecksum != null
&& !preCalculatedChecksum.isEmpty()
&& preCalculatedAlgorithm != null
&& preCalculatedAlgorithm.equalsIgnoreCase(checksumAlgorithm);

if (hasValidPreCalculatedChecksum) {
// File has pre-calculated checksum - just copy data without calculating
LOGGER.debug("Using pre-calculated checksum for file {}", zipEntryInfo.getFilePath());
final byte[] buffer = new byte[BUFFER_SIZE];
int numRead;
do {
numRead = is.read(buffer);
if (numRead > 0) {
os.write(buffer, 0, numRead);
}
} while (numRead != -1);
setChecksum(zipEntryInfo, preCalculatedChecksum, preCalculatedAlgorithm);
} else {
// Calculate checksum while copying
final byte[] buffer = new byte[BUFFER_SIZE];
final MessageDigest complete = MessageDigest.getInstance(checksumAlgorithm);
int numRead;
do {
numRead = is.read(buffer);
if (numRead > 0) {
complete.update(buffer, 0, numRead);
os.write(buffer, 0, numRead);
}
} while (numRead != -1);

setChecksum(zipEntryInfo, HexFormat.of().withUpperCase().formatHex(complete.digest()), checksumAlgorithm);
}
} finally {
IOUtils.closeQuietly(is);
IOUtils.closeQuietly(os);
Expand Down
36 changes: 34 additions & 2 deletions src/main/java/org/roda_project/commons_ip2/utils/METSFileTypeZipEntryInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,47 @@
public class METSFileTypeZipEntryInfo extends FileZipEntryInfo {
private FileType metsFileType;

public METSFileTypeZipEntryInfo(String name, Path filePath) {
/**
* Constructor with name and file path.
*
* @param name the entry name
* @param filePath the file path
*/
public METSFileTypeZipEntryInfo(final String name, final Path filePath) {
super(name, filePath);
}

public METSFileTypeZipEntryInfo(String name, Path filePath, FileType metsFileType) {
/**
* Constructor with name, file path, and METS file type.
*
* @param name the entry name
* @param filePath the file path
* @param metsFileType the METS file type
*/
public METSFileTypeZipEntryInfo(final String name, final Path filePath, final FileType metsFileType) {
super(name, filePath);
this.setMetsFileType(metsFileType);
}

/**
* Constructor with name, file path, METS file type, and pre-calculated checksum.
*
* @param name the entry name
* @param filePath the file path
* @param metsFileType the METS file type
* @param preCalculatedChecksum the pre-calculated checksum value
* @param checksumAlgorithm the checksum algorithm used
*/
public METSFileTypeZipEntryInfo(final String name, final Path filePath, final FileType metsFileType,
final String preCalculatedChecksum, final String checksumAlgorithm) {
super(name, filePath);
this.setMetsFileType(metsFileType);
if (preCalculatedChecksum != null && !preCalculatedChecksum.isEmpty()) {
this.setChecksum(preCalculatedChecksum);
this.setChecksumAlgorithm(checksumAlgorithm);
}
}

@Override
public void prepareEntryForZipping() {
// do nothing
Expand Down
Loading