Skip to content

Commit df0ffcb

Browse files
eduardojst10eduardojst10
authored
fixed loss of indexed Technical Metadata; prevent technical metadata file ID collision and fix technical metadata handling. Fixes #3521 (#3561)
1 parent 9256ca6 commit df0ffcb

10 files changed

Lines changed: 142 additions & 15 deletions

File tree

‎roda-common/roda-common-data/src/main/java/org/roda/core/data/utils/URNUtils.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ public static String createRodaPreservationURN(PreservationMetadataType preserva
5151
}
5252

5353
public static String createRodaTechnicalMetadataURN(String id, String instanceId, String technicalMetadataType) {
54-
return getTechnicalMetadataPrefix(technicalMetadataType, instanceId) + id;
54+
return getTechnicalMetadataPrefix(technicalMetadataType, instanceId) + id + RodaConstants.REPRESENTATION_INFORMATION_FILE_EXTENSION;
5555
}
5656

5757
public static String getTechnicalMetadataPrefix(String technicalMetadataType, String instanceId) {

‎roda-common/roda-common-utils/src/main/java/org/roda/core/util/IdUtils.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,12 @@ public static String getTransferredResourceUUID(Path relativeToBase) {
191191
return getTransferredResourceUUID(relativeToBase.toString());
192192
}
193193

194+
public static String createTechnicalMetadataFileId(String fileId, List<String> fileDirectoryPath){
195+
if(fileDirectoryPath.isEmpty()) return fileId;
196+
String pathPrefix = String.join(ID_SEPARATOR, fileDirectoryPath);
197+
return pathPrefix + ID_SEPARATOR + fileId;
198+
}
199+
194200
public static String getTransferredResourceUUID(String relativeToBase) {
195201
return IdUtils.createUUID(relativeToBase);
196202
}

‎roda-core/roda-core/src/main/java/org/roda/core/index/schema/collections/FileCollection.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -195,8 +195,11 @@ public SolrInputDocument toSolrDocument(ModelService model, File file, IndexingA
195195

196196
Long sizeInBytes = 0L;
197197

198-
SolrUtils.indexRepresentationTechnicalMetadata(model,
199-
getRepresentationTechnicalMetadata(((Info) info).aip, file.getRepresentationId()), fileId, doc);
198+
if (!file.isDirectory()) {
199+
String techMdFileId = IdUtils.createTechnicalMetadataFileId(fileId, file.getPath());
200+
SolrUtils.indexRepresentationTechnicalMetadata(model,
201+
getRepresentationTechnicalMetadata(((Info) info).aip, file.getRepresentationId()), techMdFileId, doc);
202+
}
200203

201204
// Add information from PREMIS
202205
Binary premisFile = getFilePremisFile(model, file);

‎roda-core/roda-core/src/main/java/org/roda/core/index/utils/SolrUtils.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1545,7 +1545,7 @@ public static void indexRepresentationTechnicalMetadata(ModelService model,
15451545

15461546
StoragePath storagePath = ModelUtils.getTechnicalMetadataStoragePath(techMd.getAipId(),
15471547
techMd.getRepresentationId(), Collections.singletonList(techMd.getType()),
1548-
urn + RodaConstants.REPRESENTATION_INFORMATION_FILE_EXTENSION);
1548+
urn);
15491549

15501550
Binary binary = model.getStorage().getBinary(storagePath);
15511551

‎roda-core/roda-core/src/main/java/org/roda/core/model/DefaultModelService.java‎

Lines changed: 49 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -721,6 +721,12 @@ public Binary retrieveDescriptiveMetadataBinary(String aipId, String representat
721721
return storage.getBinary(binaryPath);
722722
}
723723

724+
public Binary retrieveTechnicalMetadataBinary(String aipId, String representationId, List<String> fileDirectoryPath, String fileId)
725+
throws RequestNotValidException, GenericException, NotFoundException, AuthorizationDeniedException {
726+
StoragePath binaryPath = ModelUtils.getTechnicalMetadataStoragePath(aipId, representationId, fileDirectoryPath, fileId);
727+
return storage.getBinary(binaryPath);
728+
}
729+
724730
@Override
725731
public DescriptiveMetadata retrieveDescriptiveMetadata(String aipId, String descriptiveMetadataId)
726732
throws RequestNotValidException, GenericException, NotFoundException, AuthorizationDeniedException {
@@ -1790,20 +1796,59 @@ public void createTechnicalMetadata(String aipId, String representationId, Strin
17901796
StoragePath binaryPath = ModelUtils.getTechnicalMetadataStoragePath(aipId, representationId,
17911797
Collections.singletonList(metadataType), urn);
17921798
storage.createBinary(binaryPath, payload, false);
1793-
TechnicalMetadata techMd = new TechnicalMetadata(metadataType, aipId, representationId, metadataType);
17941799

1795-
AIP updatedAIP = null;
17961800
if (aipId != null) {
17971801
AIP aip = ResourceParseUtils.getAIPMetadata(getStorage(), aipId);
1798-
aip.addTechnicalMetadata(techMd);
1799-
updatedAIP = updateAIPMetadata(aip, createdBy);
1802+
addTechnicalMetadataToAIPMetadata(aip, representationId, metadataType, createdBy, notify);
1803+
}
1804+
}
1805+
1806+
@Override
1807+
public void updateTechnicalMetadata(String aipId, String representationId, String metadataType, String fileId,
1808+
ContentPayload payload, String createdBy, boolean notify)
1809+
throws AuthorizationDeniedException, RequestNotValidException, NotFoundException, GenericException {
1810+
RodaCoreFactory.checkIfWriteIsAllowedAndIfFalseThrowException(nodeType);
1811+
1812+
String urn = URNUtils.createRodaTechnicalMetadataURN(fileId, RODAInstanceUtils.getLocalInstanceIdentifier(),
1813+
metadataType.toLowerCase());
1814+
StoragePath binaryPath = ModelUtils.getTechnicalMetadataStoragePath(aipId, representationId,
1815+
Collections.singletonList(metadataType), urn);
1816+
storage.updateBinaryContent(binaryPath, payload, false, false, true, null);
1817+
// update technicalmetadata logic
1818+
if (aipId != null) {
1819+
AIP aip = ResourceParseUtils.getAIPMetadata(getStorage(), aipId);
1820+
List<TechnicalMetadata> techMetadataList = getTechnicalMetadata(aip, representationId);
1821+
techMetadataList.removeIf(tm -> tm.getId().equals(metadataType));
1822+
addTechnicalMetadataToAIPMetadata(aip, representationId, metadataType, createdBy, notify);
18001823
}
1824+
}
1825+
1826+
private void addTechnicalMetadataToAIPMetadata(AIP aip, String representationId, String metadataType,
1827+
String createdBy, boolean notify)
1828+
throws RequestNotValidException, GenericException, NotFoundException, AuthorizationDeniedException {
1829+
TechnicalMetadata techMd = new TechnicalMetadata(metadataType, aip.getId(), representationId, metadataType);
1830+
1831+
aip.addTechnicalMetadata(techMd);
1832+
AIP updatedAIP = updateAIPMetadata(aip, createdBy);
18011833

18021834
if (notify && updatedAIP != null) {
18031835
notifyAipUpdatedOnChanged(updatedAIP).failOnError();
18041836
}
18051837
}
18061838

1839+
private List<TechnicalMetadata> getTechnicalMetadata(AIP aip, String representationId) {
1840+
if (representationId == null) {
1841+
return new ArrayList<>();
1842+
}
1843+
Optional<Representation> oRep = aip.getRepresentations().stream()
1844+
.filter(rep -> rep.getId().equals(representationId)).findFirst();
1845+
if (oRep.isPresent()) {
1846+
return oRep.get().getTechnicalMetadata();
1847+
}
1848+
1849+
return new ArrayList<>();
1850+
}
1851+
18071852
@Override
18081853
public PreservationMetadata createPreservationMetadata(PreservationMetadataType type, String aipId,
18091854
String representationId, List<String> fileDirectoryPath, String fileId, ContentPayload payload, String username,

‎roda-core/roda-core/src/main/java/org/roda/core/model/DefaultTransactionalModelService.java‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -439,6 +439,23 @@ public Binary retrieveDescriptiveMetadataBinary(String aipId, String representat
439439
}
440440
}
441441

442+
@Override
443+
public Binary retrieveTechnicalMetadataBinary(String aipId, String representationId, List<String> fileDirectoryPath,
444+
String fileId) throws AuthorizationDeniedException, RequestNotValidException, NotFoundException, GenericException {
445+
List<TransactionalModelOperationLog> operationLogs = operationRegistry.registerOperationForTechnicalMetadata(aipId,
446+
representationId, fileDirectoryPath, fileId, OperationType.READ);
447+
448+
try {
449+
Binary binary = stagingModelService.retrieveTechnicalMetadataBinary(aipId, representationId, fileDirectoryPath,
450+
fileId);
451+
operationRegistry.updateOperationState(operationLogs, OperationState.SUCCESS);
452+
return binary;
453+
} catch (RequestNotValidException | GenericException | NotFoundException | AuthorizationDeniedException e) {
454+
operationRegistry.updateOperationState(operationLogs, OperationState.FAILURE);
455+
throw e;
456+
}
457+
}
458+
442459
@Override
443460
public DescriptiveMetadata retrieveDescriptiveMetadata(String aipId, String descriptiveMetadataId)
444461
throws RequestNotValidException, GenericException, NotFoundException, AuthorizationDeniedException {
@@ -1491,6 +1508,23 @@ public void createTechnicalMetadata(String aipId, String representationId, Strin
14911508
}
14921509
}
14931510

1511+
@Override
1512+
public void updateTechnicalMetadata(String aipId, String representationId, String metadataType, String fileId,
1513+
ContentPayload payload, String createdBy, boolean notify) throws AuthorizationDeniedException,
1514+
RequestNotValidException, AlreadyExistsException, NotFoundException, GenericException {
1515+
List<TransactionalModelOperationLog> operationLogs = operationRegistry.registerOperationForRepresentation(aipId,
1516+
representationId, OperationType.UPDATE);
1517+
try {
1518+
getModelService().updateTechnicalMetadata(aipId, representationId, metadataType, fileId, payload, createdBy,
1519+
notify);
1520+
operationRegistry.updateOperationState(operationLogs, OperationState.SUCCESS);
1521+
} catch (AuthorizationDeniedException | RequestNotValidException | AlreadyExistsException | NotFoundException
1522+
| GenericException e) {
1523+
operationRegistry.updateOperationState(operationLogs, OperationState.FAILURE);
1524+
throw e;
1525+
}
1526+
}
1527+
14941528
@Override
14951529
public PreservationMetadata createPreservationMetadata(PreservationMetadata.PreservationMetadataType type,
14961530
String aipId, List<String> fileDirectoryPath, String fileId, ContentPayload payload, String username,

‎roda-core/roda-core/src/main/java/org/roda/core/model/LiteRODAObjectFactory.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
import org.roda.core.data.v2.ip.metadata.OtherMetadata;
4343
import org.roda.core.data.v2.ip.metadata.PreservationMetadata;
4444
import org.roda.core.data.v2.ip.metadata.PreservationMetadata.PreservationMetadataType;
45+
import org.roda.core.data.v2.ip.metadata.TechnicalMetadata;
4546
import org.roda.core.data.v2.jobs.Job;
4647
import org.roda.core.data.v2.jobs.Report;
4748
import org.roda.core.data.v2.log.LogEntry;
@@ -162,6 +163,8 @@ public static <T extends IsRODAObject> Optional<LiteRODAObject> get(T object) {
162163
ret = getOtherMetadata(object);
163164
} else if (object instanceof PreservationMetadata) {
164165
ret = getPreservationMetadata(object);
166+
} else if (object instanceof TechnicalMetadata) {
167+
ret = getTechnicalMetadata(object);
165168
} else if (object instanceof IndexedPreservationEvent) {
166169
ret = getIndexedPreservationEvent(object);
167170
} else if (object instanceof DIPFile) {
@@ -215,6 +218,10 @@ public static <T extends IsRODAObject> Optional<LiteRODAObject> get(Class<T> obj
215218
if (ids.size() == 2 || ids.size() == 3) {
216219
ret = create(objectClass, ids.size(), ids);
217220
}
221+
} else if (objectClass == TechnicalMetadata.class) {
222+
if (ids.size() >= 3) {
223+
ret = create(objectClass, ids.size(), ids);
224+
}
218225
} else if (objectClass == OtherMetadata.class) {
219226
if (ids.size() == 2 || ids.size() == 3) {
220227
ret = create(objectClass, ids.size(), ids);
@@ -265,6 +272,16 @@ private static <T extends IsRODAObject> Optional<LiteRODAObject> getDescriptiveM
265272
return ret;
266273
}
267274

275+
private static <T extends IsRODAObject> Optional<LiteRODAObject> getTechnicalMetadata(T object) {
276+
Optional<LiteRODAObject> ret;
277+
TechnicalMetadata o = (TechnicalMetadata) object;
278+
List<String> list = new ArrayList<>();
279+
list.add(o.getAipId());
280+
list.add(o.getRepresentationId());
281+
list.add(o.getId());
282+
return get(TechnicalMetadata.class, list, false);
283+
}
284+
268285
private static <T extends IsRODAObject> Optional<LiteRODAObject> getOtherMetadata(T object) {
269286
Optional<LiteRODAObject> ret;
270287

‎roda-core/roda-core/src/main/java/org/roda/core/model/ModelService.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,9 @@ Binary retrieveDescriptiveMetadataBinary(String aipId, String descriptiveMetadat
164164
Binary retrieveDescriptiveMetadataBinary(String aipId, String representationId, String descriptiveMetadataId)
165165
throws RequestNotValidException, GenericException, NotFoundException, AuthorizationDeniedException;
166166

167+
Binary retrieveTechnicalMetadataBinary(String aipId, String representationId, List<String> fileDirectoryPath, String fileId)
168+
throws RequestNotValidException, GenericException, NotFoundException, AuthorizationDeniedException ;
169+
167170
DescriptiveMetadata retrieveDescriptiveMetadata(String aipId, String descriptiveMetadataId)
168171
throws RequestNotValidException, GenericException, NotFoundException, AuthorizationDeniedException;
169172

@@ -381,6 +384,10 @@ PreservationMetadata createPreservationMetadata(PreservationMetadata.Preservatio
381384
void createTechnicalMetadata(String aipId, String representationId, String metadataType, String fileId,
382385
ContentPayload payload, String createdBy, boolean notify) throws AuthorizationDeniedException,
383386
RequestNotValidException, AlreadyExistsException, NotFoundException, GenericException;
387+
388+
void updateTechnicalMetadata(String aipId, String representationId, String metadataType, String fileId,
389+
ContentPayload payload, String createdBy, boolean notify) throws AuthorizationDeniedException,
390+
RequestNotValidException, AlreadyExistsException, NotFoundException, GenericException;
384391

385392
public PreservationMetadata createPreservationMetadata(PreservationMetadataType type, String aipId,
386393
List<String> fileDirectoryPath, String fileId, ContentPayload payload, String username, boolean notify)

‎roda-core/roda-core/src/main/java/org/roda/core/transaction/TransactionalModelOperationRegistry.java‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
import org.roda.core.data.v2.ip.TransferredResource;
3434
import org.roda.core.data.v2.ip.metadata.DescriptiveMetadata;
3535
import org.roda.core.data.v2.ip.metadata.PreservationMetadata;
36+
import org.roda.core.data.v2.ip.metadata.TechnicalMetadata;
3637
import org.roda.core.data.v2.jobs.Job;
3738
import org.roda.core.data.v2.jobs.Report;
3839
import org.roda.core.data.v2.log.LogEntry;
@@ -113,6 +114,21 @@ public List<TransactionalModelOperationLog> registerOperationForDescriptiveMetad
113114
return operationLogs;
114115
}
115116

117+
public List<TransactionalModelOperationLog> registerOperationForTechnicalMetadata(String aipID,
118+
String representationId, List<String> fileDirectoryPath, String fileId, OperationType operation) {
119+
List<TransactionalModelOperationLog> operationLogs = new ArrayList<>();
120+
operationLogs.add(registerOperationForRelatedAIP(aipID, operation));
121+
List<String> list = new ArrayList<>();
122+
list.add(aipID);
123+
list.addAll(fileDirectoryPath);
124+
if (representationId != null) {
125+
list.add(representationId);
126+
}
127+
list.add(fileId);
128+
operationLogs.add(registerOperation(TechnicalMetadata.class, list, operation));
129+
return operationLogs;
130+
}
131+
116132
public List<TransactionalModelOperationLog> registerOperationForRepresentation(String aipID, String representationId,
117133
OperationType operation) {
118134
List<TransactionalModelOperationLog> operationLogs = new ArrayList<>();
@@ -511,7 +527,6 @@ public <T extends IsRODAObject> void releaseLock(Class<T> objectClass, String id
511527
"[transactionId:" + transaction.getId() + "] Object class is not lockable: " + objectClass.getName());
512528
}
513529

514-
515530
Optional<LiteRODAObject> liteRODAObject = LiteRODAObjectFactory.get(objectClass, id);
516531
if (liteRODAObject.isPresent()) {
517532
String lite = liteRODAObject.get().getInfo();

‎roda-ui/roda-wui/src/main/java/org/roda/wui/api/v2/services/FilesService.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -392,18 +392,18 @@ public StreamResponse retrieveFileTechnicalMetadataHTML(RequestContext requestCo
392392
NotFoundException, GenericException, TechnicalMetadataNotFoundException {
393393
ModelService model = requestContext.getModelService();
394394
Representation representation = model.retrieveRepresentation(file.getAipId(), file.getRepresentationId());
395-
String techMDURN = URNUtils.createRodaTechnicalMetadataURN(file.getId(),
395+
String techMDURN = URNUtils.createRodaTechnicalMetadataURN(IdUtils.createTechnicalMetadataFileId(file.getId(), file.getPath()),
396396
RODAInstanceUtils.getLocalInstanceIdentifier(), type.toLowerCase());
397397
Binary metadataBinary;
398398
if (versionID != null) {
399399
BinaryVersion binaryVersion = model.getBinaryVersion(representation, versionID,
400400
List.of(RodaConstants.STORAGE_DIRECTORY_METADATA, RodaConstants.STORAGE_DIRECTORY_TECHNICAL, type,
401-
techMDURN + RodaConstants.REPRESENTATION_INFORMATION_FILE_EXTENSION));
401+
techMDURN));
402402
metadataBinary = binaryVersion.getBinary();
403403
} else {
404404
metadataBinary = model.getBinary(representation, RodaConstants.STORAGE_DIRECTORY_METADATA,
405405
RodaConstants.STORAGE_DIRECTORY_TECHNICAL, type,
406-
techMDURN + RodaConstants.REPRESENTATION_INFORMATION_FILE_EXTENSION);
406+
techMDURN);
407407
}
408408
String filename = metadataBinary.getStoragePath().getName() + HTML_EXT;
409409
String htmlDescriptive = HTMLUtils.technicalMetadataToHtml(metadataBinary, type, versionID,
@@ -427,18 +427,18 @@ public StreamResponse retrieveFileTechnicalMetadata(RequestContext requestContex
427427
StreamResponse ret;
428428
ModelService model = requestContext.getModelService();
429429
Representation representation = model.retrieveRepresentation(file.getAipId(), file.getRepresentationId());
430-
String techMDURN = URNUtils.createRodaTechnicalMetadataURN(file.getId(),
430+
String techMDURN = URNUtils.createRodaTechnicalMetadataURN(IdUtils.createTechnicalMetadataFileId(file.getId(), file.getPath()),
431431
RODAInstanceUtils.getLocalInstanceIdentifier(), type.toLowerCase());
432432
Binary metadataBinary;
433433
if (versionID != null) {
434434
BinaryVersion binaryVersion = model.getBinaryVersion(representation, versionID,
435435
List.of(RodaConstants.STORAGE_DIRECTORY_METADATA, RodaConstants.STORAGE_DIRECTORY_TECHNICAL, type,
436-
techMDURN + RodaConstants.REPRESENTATION_INFORMATION_FILE_EXTENSION));
436+
techMDURN));
437437
metadataBinary = binaryVersion.getBinary();
438438
} else {
439439
metadataBinary = model.getBinary(representation, RodaConstants.STORAGE_DIRECTORY_METADATA,
440440
RodaConstants.STORAGE_DIRECTORY_TECHNICAL, type,
441-
techMDURN + RodaConstants.REPRESENTATION_INFORMATION_FILE_EXTENSION);
441+
techMDURN);
442442
}
443443
stream = new BinaryConsumesOutputStream(metadataBinary, RodaConstants.MEDIA_TYPE_TEXT_XML);
444444

0 commit comments

Comments
 (0)