From f8524baf396d76820c98232316d74083f3c1f49b Mon Sep 17 00:00:00 2001 From: Hugues <hugues.cazeaux@unige.ch> Date: Tue, 21 Jan 2025 21:44:42 +0100 Subject: [PATCH 01/18] feat: step-1 --- .../src/main/java/ch/dlcm/job/MetadataMigrationJob.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DLCM-PreservationPlanning/src/main/java/ch/dlcm/job/MetadataMigrationJob.java b/DLCM-PreservationPlanning/src/main/java/ch/dlcm/job/MetadataMigrationJob.java index 6e4cb3bb17..dc9b857ade 100644 --- a/DLCM-PreservationPlanning/src/main/java/ch/dlcm/job/MetadataMigrationJob.java +++ b/DLCM-PreservationPlanning/src/main/java/ch/dlcm/job/MetadataMigrationJob.java @@ -46,6 +46,7 @@ import ch.dlcm.service.rest.trusted.TrustedDepositRemoteResourceService; import ch.dlcm.service.rest.trusted.TrustedSubmissionInfoPackageRemoteResourceService; public class MetadataMigrationJob extends AbstractJob { + private static final String ARCHIVE_NOT_UPGRADABLE = "Archive not upgradable because it is already in default version or the data are not cleaned"; private final TrustedDepositRemoteResourceService depositRemoteResourceService; @@ -103,7 +104,6 @@ public class MetadataMigrationJob extends AbstractJob { job = this.jobProcessingService.saveStep(job, executionId, num, total); } } while (collection.getPage().hasNext()); - } private PreservationJob ignoreArchive(PreservationJob job, String executionId, String storageUrl, ArchivalInfoPackage aip, -- GitLab From 7d3e2be2a5ccdad910723ae26e41a17a6450e08f Mon Sep 17 00:00:00 2001 From: Hugues <hugues.cazeaux@unige.ch> Date: Wed, 22 Jan 2025 15:41:35 +0100 Subject: [PATCH 02/18] feat: step-2 --- .../src/main/java/ch/dlcm/job/MetadataMigrationJob.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/DLCM-PreservationPlanning/src/main/java/ch/dlcm/job/MetadataMigrationJob.java b/DLCM-PreservationPlanning/src/main/java/ch/dlcm/job/MetadataMigrationJob.java index dc9b857ade..2a2aec99d3 100644 --- a/DLCM-PreservationPlanning/src/main/java/ch/dlcm/job/MetadataMigrationJob.java +++ b/DLCM-PreservationPlanning/src/main/java/ch/dlcm/job/MetadataMigrationJob.java @@ -71,6 +71,7 @@ public class MetadataMigrationJob extends AbstractJob { @Override protected void execute(PreservationJob job, String executionId) { + final long total = this.getTotal(this.defaultStorageUrl); Pageable pageable = this.getPageRequest(); RestCollection<ArchivalInfoPackage> collection; @@ -95,7 +96,8 @@ public class MetadataMigrationJob extends AbstractJob { job = this.ignoreArchive(job, executionId, this.defaultStorageUrl, aip, sipList, depositList); } } catch (final Exception e) { - job = this.addInErrorItem(job, executionId, aip.getResId(), this.getAipPublicUrl(this.defaultStorageUrl, aip.getResId()), e.getMessage()); + job = this.addInErrorItem(job, executionId, aip.getResId(), this.getAipPublicUrl(this.defaultStorageUrl, aip.getResId()), + e.getMessage()); } // Save intermediate step by 10% final long num = job.getLastExecution().getLastExecutionReport().getProcessedItems() -- GitLab From b0b1d610a1e2cf201a63430542fd86e34d63fbab Mon Sep 17 00:00:00 2001 From: Hugues <hugues.cazeaux@unige.ch> Date: Wed, 22 Jan 2025 17:14:46 +0100 Subject: [PATCH 03/18] feat: step-3 --- .../src/main/java/ch/dlcm/controller/ingest/SIPController.java | 1 + 1 file changed, 1 insertion(+) diff --git a/DLCM-Ingest/src/main/java/ch/dlcm/controller/ingest/SIPController.java b/DLCM-Ingest/src/main/java/ch/dlcm/controller/ingest/SIPController.java index 59e0fe95ad..ea69099a93 100644 --- a/DLCM-Ingest/src/main/java/ch/dlcm/controller/ingest/SIPController.java +++ b/DLCM-Ingest/src/main/java/ch/dlcm/controller/ingest/SIPController.java @@ -314,6 +314,7 @@ public class SIPController extends AbstractPackageController<SipDataFile, Submis SubmissionInfoPackage sip = this.itemService.findOne(id); final PackageStatus currentStatus = sip.getInfo().getStatus(); final Result result = new Result(id); + if ((newStatus == EDITING_METADATA && (currentStatus == PackageStatus.COMPLETED || currentStatus == PackageStatus.CLEANED)) || (newStatus == PackageStatus.UPGRADING_METADATA && currentStatus == PackageStatus.CLEANED -- GitLab From f7b33deb2c1fd709181f5f4a5baa094a54dc8eac Mon Sep 17 00:00:00 2001 From: Hugues <hugues.cazeaux@unige.ch> Date: Fri, 24 Jan 2025 17:03:21 +0100 Subject: [PATCH 04/18] feat: step-5 --- .../src/main/java/ch/dlcm/DLCMMetadataVersion.java | 2 +- .../java/ch/dlcm/model/tool/MetadataMigrationTool.java | 10 +++++----- .../src/main/resources/xslt/dlcm4migration-1.0-1.1.xsl | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/DLCM-Model/src/main/java/ch/dlcm/DLCMMetadataVersion.java b/DLCM-Model/src/main/java/ch/dlcm/DLCMMetadataVersion.java index b8475d371c..411f4f489d 100644 --- a/DLCM-Model/src/main/java/ch/dlcm/DLCMMetadataVersion.java +++ b/DLCM-Model/src/main/java/ch/dlcm/DLCMMetadataVersion.java @@ -44,7 +44,7 @@ import ch.unige.solidify.validation.ValidationError; - 4.0 = Datacite 4.5 + PREMIS 3.0 + DLCM Info 3.0 + Update of Archive Thumbnail & DUA & README """) public enum DLCMMetadataVersion { - V1_0("1.0", 10), // v1.0: DataCite 4.0 + Premis 3.0 + DLCM Info 1.0 + V1_0("1.0", 10), // v1.0: DataCite 4.0 + Premis 3.0 V1_1("1.1", 11), // v1.1: DataCite 4.0 + Premis 3.0 + DLCM Info 1.0 V2_0("2.0", 20), // v2.0: DataCite 4.3 + Premis 3.0 + DLCM Info 2.0 + Data File Categories V2_1("2.1", 21), // v2.1: DataCite 4.3 + Premis 3.0 + DLCM Info 2.1 + Dataset Thumbnail support diff --git a/DLCM-Model/src/main/java/ch/dlcm/model/tool/MetadataMigrationTool.java b/DLCM-Model/src/main/java/ch/dlcm/model/tool/MetadataMigrationTool.java index 771c9cf45d..ff94d38e67 100644 --- a/DLCM-Model/src/main/java/ch/dlcm/model/tool/MetadataMigrationTool.java +++ b/DLCM-Model/src/main/java/ch/dlcm/model/tool/MetadataMigrationTool.java @@ -25,9 +25,9 @@ package ch.dlcm.model.tool; import java.io.IOException; import java.net.URL; -import java.nio.file.Path; +import java.nio.file.Path;<<<<<<<HEAD import java.time.OffsetDateTime; -import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatter;=======>>>>>>>f233b1b66(feat:step-5) import java.util.ArrayList; import java.util.EnumMap; import java.util.List; @@ -35,11 +35,11 @@ import java.util.Map; import org.springframework.core.io.ClassPathResource; -import ch.unige.solidify.SolidifyConstants; +import ch.unige.solidify.SolidifyConstants;<<<<<<<HEAD import ch.unige.solidify.exception.SolidifyProcessingException; import ch.unige.solidify.util.FileTool; import ch.unige.solidify.util.StringTool; -import ch.unige.solidify.util.XMLTool; +import ch.unige.solidify.util.XMLTool;=======>>>>>>>f233b1b66(feat:step-5) import ch.dlcm.DLCMConstants; import ch.dlcm.DLCMMetadataVersion; @@ -56,7 +56,7 @@ public class MetadataMigrationTool { if (!FileTool.copyFile(metadataXml, originalXml)) { throw new SolidifyProcessingException("Cannot copy the original metadata file: " + metadataXml + " => " + originalXml); } - // Migrate the metadat + // Migrate the metadata DLCMMetadataVersion fromVersion = version; do { final DLCMMetadataVersion toVersion = MetadataMigrationTool.getMigrationPath().get(fromVersion); diff --git a/DLCM-Model/src/main/resources/xslt/dlcm4migration-1.0-1.1.xsl b/DLCM-Model/src/main/resources/xslt/dlcm4migration-1.0-1.1.xsl index cd37d0fdbb..5a95219c97 100644 --- a/DLCM-Model/src/main/resources/xslt/dlcm4migration-1.0-1.1.xsl +++ b/DLCM-Model/src/main/resources/xslt/dlcm4migration-1.0-1.1.xsl @@ -6,7 +6,7 @@ <xsl:output method="xml" omit-xml-declaration="no" indent="yes" encoding="UTF-8" /> <!-- Browse all nodes --> - <xsl:template match="@* | node()" priority="-1"> + <xsl:template match="@* | node()"> <xsl:copy> <xsl:apply-templates select="@* | node()" /> </xsl:copy> -- GitLab From 4ee1348a4243b359f922e04781e1e3a86a43810b Mon Sep 17 00:00:00 2001 From: Hugues <hugues.cazeaux@unige.ch> Date: Fri, 24 Jan 2025 18:32:48 +0100 Subject: [PATCH 05/18] feat: step-6 --- .../java/ch/dlcm/model/tool/MetadataMigrationTool.java | 8 ++++---- .../src/main/resources/xslt/dlcm4migration-2.0-2.1.xsl | 5 +++++ .../src/main/resources/xslt/dlcm4migration-3.0-3.1.xsl | 2 +- .../src/main/resources/xslt/dlcm4migration-3.1-4.0.xsl | 5 +++-- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/DLCM-Model/src/main/java/ch/dlcm/model/tool/MetadataMigrationTool.java b/DLCM-Model/src/main/java/ch/dlcm/model/tool/MetadataMigrationTool.java index ff94d38e67..d11848bd2e 100644 --- a/DLCM-Model/src/main/java/ch/dlcm/model/tool/MetadataMigrationTool.java +++ b/DLCM-Model/src/main/java/ch/dlcm/model/tool/MetadataMigrationTool.java @@ -25,9 +25,9 @@ package ch.dlcm.model.tool; import java.io.IOException; import java.net.URL; -import java.nio.file.Path;<<<<<<<HEAD +import java.nio.file.Path; import java.time.OffsetDateTime; -import java.time.format.DateTimeFormatter;=======>>>>>>>f233b1b66(feat:step-5) +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.EnumMap; import java.util.List; @@ -35,11 +35,11 @@ import java.util.Map; import org.springframework.core.io.ClassPathResource; -import ch.unige.solidify.SolidifyConstants;<<<<<<<HEAD +import ch.unige.solidify.SolidifyConstants; import ch.unige.solidify.exception.SolidifyProcessingException; import ch.unige.solidify.util.FileTool; import ch.unige.solidify.util.StringTool; -import ch.unige.solidify.util.XMLTool;=======>>>>>>>f233b1b66(feat:step-5) +import ch.unige.solidify.util.XMLTool; import ch.dlcm.DLCMConstants; import ch.dlcm.DLCMMetadataVersion; diff --git a/DLCM-Model/src/main/resources/xslt/dlcm4migration-2.0-2.1.xsl b/DLCM-Model/src/main/resources/xslt/dlcm4migration-2.0-2.1.xsl index 5a2762ba73..172ffdf15a 100644 --- a/DLCM-Model/src/main/resources/xslt/dlcm4migration-2.0-2.1.xsl +++ b/DLCM-Model/src/main/resources/xslt/dlcm4migration-2.0-2.1.xsl @@ -11,6 +11,11 @@ <xsl:apply-templates select="@* | node()" /> </xsl:copy> </xsl:template> + + <!-- METS Profile --> + <xsl:template match="mets:mets/@PROFILE"> + <xsl:attribute name="PROFILE">dlcm_profile-2.1.xml</xsl:attribute> + </xsl:template> <!-- METS Profile --> <xsl:template match="mets:mets/@PROFILE"> diff --git a/DLCM-Model/src/main/resources/xslt/dlcm4migration-3.0-3.1.xsl b/DLCM-Model/src/main/resources/xslt/dlcm4migration-3.0-3.1.xsl index 359a7ea7c2..d8a471639b 100644 --- a/DLCM-Model/src/main/resources/xslt/dlcm4migration-3.0-3.1.xsl +++ b/DLCM-Model/src/main/resources/xslt/dlcm4migration-3.0-3.1.xsl @@ -11,7 +11,7 @@ <xsl:apply-templates select="@* | node()" /> </xsl:copy> </xsl:template> - + <!-- METS Profile --> <xsl:template match="mets:mets/@PROFILE"> <xsl:attribute name="PROFILE"><xsl:text>dlcm_profile-3.1.xml</xsl:text></xsl:attribute> diff --git a/DLCM-Model/src/main/resources/xslt/dlcm4migration-3.1-4.0.xsl b/DLCM-Model/src/main/resources/xslt/dlcm4migration-3.1-4.0.xsl index 3acefd8874..2ca79a60b1 100644 --- a/DLCM-Model/src/main/resources/xslt/dlcm4migration-3.1-4.0.xsl +++ b/DLCM-Model/src/main/resources/xslt/dlcm4migration-3.1-4.0.xsl @@ -1,7 +1,8 @@ <?xml version="1.0" encoding="UTF-8"?> <xsl:stylesheet version="2.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:mets="http://www.loc.gov/METS/" xmlns:datacite="http://datacite.org/schema/kernel-4" xmlns:premis="http://www.loc.gov/premis/v3" - xmlns:dlcm2="http://www.dlcm.ch/dlcm/v2" xmlns:dlcm="http://www.dlcm.ch/dlcm/v3" xmlns:fits="http://hul.harvard.edu/ois/xml/ns/fits/fits_output" exclude-result-prefixes=""> + xmlns:dlcm2="http://www.dlcm.ch/dlcm/v2" xmlns:dlcm="http://www.dlcm.ch/dlcm/v3" xmlns:fits="http://hul.harvard.edu/ois/xml/ns/fits/fits_output" + exclude-result-prefixes=""> <xsl:output method="xml" omit-xml-declaration="no" indent="yes" encoding="UTF-8" /> @@ -11,7 +12,7 @@ <xsl:apply-templates select="@* | node()" /> </xsl:copy> </xsl:template> - + <!-- Migrate namespaces --> <xsl:template match="*"> <xsl:element name="{name()}" namespace="{namespace-uri(.)}"> -- GitLab From 42d5095fd7889a217b2b8049f7ced888117844f8 Mon Sep 17 00:00:00 2001 From: Hugues <hugues.cazeaux@unige.ch> Date: Wed, 29 Jan 2025 15:48:18 +0100 Subject: [PATCH 06/18] feat: step-9 --- .../src/main/resources/xslt/dlcm4migration-1.0-1.1.xsl | 5 +++++ .../src/main/resources/xslt/dlcm4migration-1.1-2.0.xsl | 5 +++++ .../src/main/resources/xslt/dlcm4migration-2.1-3.0.xsl | 5 +++++ .../src/main/resources/xslt/dlcm4migration-3.1-4.0.xsl | 5 +++++ 4 files changed, 20 insertions(+) diff --git a/DLCM-Model/src/main/resources/xslt/dlcm4migration-1.0-1.1.xsl b/DLCM-Model/src/main/resources/xslt/dlcm4migration-1.0-1.1.xsl index 5a95219c97..fb065b0f35 100644 --- a/DLCM-Model/src/main/resources/xslt/dlcm4migration-1.0-1.1.xsl +++ b/DLCM-Model/src/main/resources/xslt/dlcm4migration-1.0-1.1.xsl @@ -21,5 +21,10 @@ <xsl:template match="mets:mets/mets:dmdSec/mets:mdWrap/@MDTYPEVERSION"> <xsl:attribute name="MDTYPEVERSION"><xsl:text>DataCite Metadata Schema 4.0 (dlcm_datacite-1.0.xsd)</xsl:text></xsl:attribute> </xsl:template> + + <!-- METS MDTYPEVERSION --> + <xsl:template match="mets:mets/mets:dmdSec/mets:mdWrap/@MDTYPEVERSION"> + <xsl:attribute name="MDTYPEVERSION">DataCite Metadata Schema 4.0 (dlcm_datacite-1.0.xsd)</xsl:attribute> + </xsl:template> </xsl:stylesheet> diff --git a/DLCM-Model/src/main/resources/xslt/dlcm4migration-1.1-2.0.xsl b/DLCM-Model/src/main/resources/xslt/dlcm4migration-1.1-2.0.xsl index 0a079b79d5..5317531c36 100644 --- a/DLCM-Model/src/main/resources/xslt/dlcm4migration-1.1-2.0.xsl +++ b/DLCM-Model/src/main/resources/xslt/dlcm4migration-1.1-2.0.xsl @@ -70,5 +70,10 @@ <xsl:value-of select="." /> </xsl:element> </xsl:template> + + <!-- METS MDTYPEVERSION --> + <xsl:template match="mets:mets/mets:dmdSec/mets:mdWrap/@MDTYPEVERSION"> + <xsl:attribute name="MDTYPEVERSION">DataCite Metadata Schema 4.3 (dlcm_datacite-2.0.xsd)</xsl:attribute> + </xsl:template> </xsl:stylesheet> diff --git a/DLCM-Model/src/main/resources/xslt/dlcm4migration-2.1-3.0.xsl b/DLCM-Model/src/main/resources/xslt/dlcm4migration-2.1-3.0.xsl index c3330ebb15..03a5baa5d8 100644 --- a/DLCM-Model/src/main/resources/xslt/dlcm4migration-2.1-3.0.xsl +++ b/DLCM-Model/src/main/resources/xslt/dlcm4migration-2.1-3.0.xsl @@ -22,4 +22,9 @@ <xsl:attribute name="MDTYPEVERSION"><xsl:text>DataCite Metadata Schema 4.4 (dlcm_datacite-3.0.xsd)</xsl:text></xsl:attribute> </xsl:template> + <!-- METS MDTYPEVERSION --> + <xsl:template match="mets:mets/mets:dmdSec/mets:mdWrap/@MDTYPEVERSION"> + <xsl:attribute name="MDTYPEVERSION">DataCite Metadata Schema 4.4 (dlcm_datacite-3.0.xsd)</xsl:attribute> + </xsl:template> + </xsl:stylesheet> diff --git a/DLCM-Model/src/main/resources/xslt/dlcm4migration-3.1-4.0.xsl b/DLCM-Model/src/main/resources/xslt/dlcm4migration-3.1-4.0.xsl index 2ca79a60b1..2d04fb202f 100644 --- a/DLCM-Model/src/main/resources/xslt/dlcm4migration-3.1-4.0.xsl +++ b/DLCM-Model/src/main/resources/xslt/dlcm4migration-3.1-4.0.xsl @@ -65,5 +65,10 @@ <xsl:template match="mets:mets/mets:dmdSec/mets:mdWrap/@MDTYPEVERSION"> <xsl:attribute name="MDTYPEVERSION"><xsl:text>DataCite Metadata Schema 4.5 (dlcm_datacite-4.0.xsd)</xsl:text></xsl:attribute> </xsl:template> + + <!-- METS MDTYPEVERSION --> + <xsl:template match="mets:mets/mets:dmdSec/mets:mdWrap/@MDTYPEVERSION"> + <xsl:attribute name="MDTYPEVERSION">DataCite Metadata Schema 4.5 (dlcm_datacite-4.0.xsd)</xsl:attribute> + </xsl:template> </xsl:stylesheet> -- GitLab From e0e9dfc016e259af221ad32cb537ac8dc95040b8 Mon Sep 17 00:00:00 2001 From: Hugues <hugues.cazeaux@unige.ch> Date: Fri, 31 Jan 2025 20:01:34 +0100 Subject: [PATCH 07/18] feat: step-10 --- .../src/main/java/ch/dlcm/DLCMMetadataVersion.java | 2 +- .../main/resources/xslt/dlcm4migration-1.0-1.1.xsl | 7 +------ .../main/resources/xslt/dlcm4migration-1.1-2.0.xsl | 5 ----- .../main/resources/xslt/dlcm4migration-2.0-2.1.xsl | 9 +++++++-- .../main/resources/xslt/dlcm4migration-2.1-3.0.xsl | 5 ----- .../main/resources/xslt/dlcm4migration-3.0-3.1.xsl | 2 +- .../main/resources/xslt/dlcm4migration-3.1-4.0.xsl | 12 +----------- 7 files changed, 11 insertions(+), 31 deletions(-) diff --git a/DLCM-Model/src/main/java/ch/dlcm/DLCMMetadataVersion.java b/DLCM-Model/src/main/java/ch/dlcm/DLCMMetadataVersion.java index 411f4f489d..b8475d371c 100644 --- a/DLCM-Model/src/main/java/ch/dlcm/DLCMMetadataVersion.java +++ b/DLCM-Model/src/main/java/ch/dlcm/DLCMMetadataVersion.java @@ -44,7 +44,7 @@ import ch.unige.solidify.validation.ValidationError; - 4.0 = Datacite 4.5 + PREMIS 3.0 + DLCM Info 3.0 + Update of Archive Thumbnail & DUA & README """) public enum DLCMMetadataVersion { - V1_0("1.0", 10), // v1.0: DataCite 4.0 + Premis 3.0 + V1_0("1.0", 10), // v1.0: DataCite 4.0 + Premis 3.0 + DLCM Info 1.0 V1_1("1.1", 11), // v1.1: DataCite 4.0 + Premis 3.0 + DLCM Info 1.0 V2_0("2.0", 20), // v2.0: DataCite 4.3 + Premis 3.0 + DLCM Info 2.0 + Data File Categories V2_1("2.1", 21), // v2.1: DataCite 4.3 + Premis 3.0 + DLCM Info 2.1 + Dataset Thumbnail support diff --git a/DLCM-Model/src/main/resources/xslt/dlcm4migration-1.0-1.1.xsl b/DLCM-Model/src/main/resources/xslt/dlcm4migration-1.0-1.1.xsl index fb065b0f35..cd37d0fdbb 100644 --- a/DLCM-Model/src/main/resources/xslt/dlcm4migration-1.0-1.1.xsl +++ b/DLCM-Model/src/main/resources/xslt/dlcm4migration-1.0-1.1.xsl @@ -6,7 +6,7 @@ <xsl:output method="xml" omit-xml-declaration="no" indent="yes" encoding="UTF-8" /> <!-- Browse all nodes --> - <xsl:template match="@* | node()"> + <xsl:template match="@* | node()" priority="-1"> <xsl:copy> <xsl:apply-templates select="@* | node()" /> </xsl:copy> @@ -21,10 +21,5 @@ <xsl:template match="mets:mets/mets:dmdSec/mets:mdWrap/@MDTYPEVERSION"> <xsl:attribute name="MDTYPEVERSION"><xsl:text>DataCite Metadata Schema 4.0 (dlcm_datacite-1.0.xsd)</xsl:text></xsl:attribute> </xsl:template> - - <!-- METS MDTYPEVERSION --> - <xsl:template match="mets:mets/mets:dmdSec/mets:mdWrap/@MDTYPEVERSION"> - <xsl:attribute name="MDTYPEVERSION">DataCite Metadata Schema 4.0 (dlcm_datacite-1.0.xsd)</xsl:attribute> - </xsl:template> </xsl:stylesheet> diff --git a/DLCM-Model/src/main/resources/xslt/dlcm4migration-1.1-2.0.xsl b/DLCM-Model/src/main/resources/xslt/dlcm4migration-1.1-2.0.xsl index 5317531c36..0a079b79d5 100644 --- a/DLCM-Model/src/main/resources/xslt/dlcm4migration-1.1-2.0.xsl +++ b/DLCM-Model/src/main/resources/xslt/dlcm4migration-1.1-2.0.xsl @@ -70,10 +70,5 @@ <xsl:value-of select="." /> </xsl:element> </xsl:template> - - <!-- METS MDTYPEVERSION --> - <xsl:template match="mets:mets/mets:dmdSec/mets:mdWrap/@MDTYPEVERSION"> - <xsl:attribute name="MDTYPEVERSION">DataCite Metadata Schema 4.3 (dlcm_datacite-2.0.xsd)</xsl:attribute> - </xsl:template> </xsl:stylesheet> diff --git a/DLCM-Model/src/main/resources/xslt/dlcm4migration-2.0-2.1.xsl b/DLCM-Model/src/main/resources/xslt/dlcm4migration-2.0-2.1.xsl index 172ffdf15a..48090c72a2 100644 --- a/DLCM-Model/src/main/resources/xslt/dlcm4migration-2.0-2.1.xsl +++ b/DLCM-Model/src/main/resources/xslt/dlcm4migration-2.0-2.1.xsl @@ -11,10 +11,15 @@ <xsl:apply-templates select="@* | node()" /> </xsl:copy> </xsl:template> - + <!-- METS Profile --> <xsl:template match="mets:mets/@PROFILE"> - <xsl:attribute name="PROFILE">dlcm_profile-2.1.xml</xsl:attribute> + <xsl:attribute name="PROFILE"><xsl:text>dlcm_profile-2.1.xml</xsl:text></xsl:attribute> + </xsl:template> + + <!-- METS MDTYPEVERSION --> + <xsl:template match="mets:mets/mets:dmdSec/mets:mdWrap/@MDTYPEVERSION"> + <xsl:attribute name="MDTYPEVERSION"><xsl:text>DataCite Metadata Schema 4.3 (dlcm_datacite-2.0.xsd)</xsl:text></xsl:attribute> </xsl:template> <!-- METS Profile --> diff --git a/DLCM-Model/src/main/resources/xslt/dlcm4migration-2.1-3.0.xsl b/DLCM-Model/src/main/resources/xslt/dlcm4migration-2.1-3.0.xsl index 03a5baa5d8..c3330ebb15 100644 --- a/DLCM-Model/src/main/resources/xslt/dlcm4migration-2.1-3.0.xsl +++ b/DLCM-Model/src/main/resources/xslt/dlcm4migration-2.1-3.0.xsl @@ -22,9 +22,4 @@ <xsl:attribute name="MDTYPEVERSION"><xsl:text>DataCite Metadata Schema 4.4 (dlcm_datacite-3.0.xsd)</xsl:text></xsl:attribute> </xsl:template> - <!-- METS MDTYPEVERSION --> - <xsl:template match="mets:mets/mets:dmdSec/mets:mdWrap/@MDTYPEVERSION"> - <xsl:attribute name="MDTYPEVERSION">DataCite Metadata Schema 4.4 (dlcm_datacite-3.0.xsd)</xsl:attribute> - </xsl:template> - </xsl:stylesheet> diff --git a/DLCM-Model/src/main/resources/xslt/dlcm4migration-3.0-3.1.xsl b/DLCM-Model/src/main/resources/xslt/dlcm4migration-3.0-3.1.xsl index d8a471639b..359a7ea7c2 100644 --- a/DLCM-Model/src/main/resources/xslt/dlcm4migration-3.0-3.1.xsl +++ b/DLCM-Model/src/main/resources/xslt/dlcm4migration-3.0-3.1.xsl @@ -11,7 +11,7 @@ <xsl:apply-templates select="@* | node()" /> </xsl:copy> </xsl:template> - + <!-- METS Profile --> <xsl:template match="mets:mets/@PROFILE"> <xsl:attribute name="PROFILE"><xsl:text>dlcm_profile-3.1.xml</xsl:text></xsl:attribute> diff --git a/DLCM-Model/src/main/resources/xslt/dlcm4migration-3.1-4.0.xsl b/DLCM-Model/src/main/resources/xslt/dlcm4migration-3.1-4.0.xsl index 2d04fb202f..7b4a75988c 100644 --- a/DLCM-Model/src/main/resources/xslt/dlcm4migration-3.1-4.0.xsl +++ b/DLCM-Model/src/main/resources/xslt/dlcm4migration-3.1-4.0.xsl @@ -12,7 +12,7 @@ <xsl:apply-templates select="@* | node()" /> </xsl:copy> </xsl:template> - + <!-- Migrate namespaces --> <xsl:template match="*"> <xsl:element name="{name()}" namespace="{namespace-uri(.)}"> @@ -56,19 +56,9 @@ <xsl:attribute name="PROFILE"><xsl:text>dlcm_profile-4.0.xml</xsl:text></xsl:attribute> </xsl:template> - <!-- Migrate METS xmlns:dlcm --> - <xsl:template match="mets:mets/@dlcm"> - <xsl:attribute name="dlcm">http://www.dlcm.ch/dlcm/v3</xsl:attribute> - </xsl:template> - <!-- Migrate METS MDTYPEVERSION --> <xsl:template match="mets:mets/mets:dmdSec/mets:mdWrap/@MDTYPEVERSION"> <xsl:attribute name="MDTYPEVERSION"><xsl:text>DataCite Metadata Schema 4.5 (dlcm_datacite-4.0.xsd)</xsl:text></xsl:attribute> </xsl:template> - - <!-- METS MDTYPEVERSION --> - <xsl:template match="mets:mets/mets:dmdSec/mets:mdWrap/@MDTYPEVERSION"> - <xsl:attribute name="MDTYPEVERSION">DataCite Metadata Schema 4.5 (dlcm_datacite-4.0.xsd)</xsl:attribute> - </xsl:template> </xsl:stylesheet> -- GitLab From d6fd373250bec34fe87d56bbd5117d5c8a09a297 Mon Sep 17 00:00:00 2001 From: Hugues <hugues.cazeaux@unige.ch> Date: Tue, 4 Feb 2025 17:15:59 +0100 Subject: [PATCH 08/18] feat: file format service in metadata service --- .../src/main/java/ch/dlcm/service/MetadataService.java | 7 ++++--- .../java/ch/dlcm/service/metadata/MetadataVersion4.java | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/MetadataService.java b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/MetadataService.java index 27761b73df..8400e62335 100644 --- a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/MetadataService.java +++ b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/MetadataService.java @@ -97,6 +97,9 @@ import ch.dlcm.service.rest.trusted.TrustedDepositRemoteResourceService; @Service public class MetadataService extends DLCMService implements OAIMetadataService { + + private static final Logger log = LoggerFactory.getLogger(MetadataService.class); + private final String preingestTempLocation; private final DLCMProperties dlcmProperties; private final DLCMRepositoryDescription repositoryDescription; @@ -113,9 +116,7 @@ public class MetadataService extends DLCMService implements OAIMetadataService { private final FallbackPreservationPolicyRemoteResourceService preservationPolicyRemoteService; private final FallbackArchiveTypeRemoteResourceService archiveTypeResourceService; private final FileFormatService fileFormatService; - private TrustedDepositRemoteResourceService depositRemoteService; - - private static final Logger log = LoggerFactory.getLogger(MetadataService.class); + private final TrustedDepositRemoteResourceService depositRemoteService; public MetadataService( FallbackLanguageRemoteResourceService languageRemoteService, diff --git a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion4.java b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion4.java index 025528af9d..3dfd90b119 100644 --- a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion4.java +++ b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion4.java @@ -217,7 +217,9 @@ public class MetadataVersion4 extends MetadataGenerator { archivePrivateMetadataResourceService, metadataTypeService, orgUnitResourceService, licenseRemoteResourceService, languageResourceService, personRemoteResourceService, preservationPolicyRemoteService, archiveTypeResourceService, fileFormatService, historyService, gitInfoProperties); + this.depositRemoteService = depositRemoteService; + // JAXB context try { this.jaxbContext = JAXBContext.newInstance(Mets.class, Resource.class, PremisComplexType.class, Fits.class); -- GitLab From f8c5121fa1cd07aec85a6b6266fffa666b87edce Mon Sep 17 00:00:00 2001 From: Hugues <hugues.cazeaux@unige.ch> Date: Wed, 5 Feb 2025 18:08:54 +0100 Subject: [PATCH 09/18] feat: step-1 --- .../archivalstorage/AIPController.java | 6 ++ .../ch/dlcm/service/AipStatusService.java | 25 +++++ .../resources/locale/messages_dlcm.properties | 2 + .../java/ch/dlcm/model/PackageStatus.java | 6 ++ .../ch/dlcm/model/preservation/JobType.java | 1 + .../java/ch/dlcm/rest/DLCMActionName.java | 1 + .../ch/dlcm/job/ComplianceLevelUpdateJob.java | 91 +++++++++++++++++++ .../ch/dlcm/job/MetadataMigrationJob.java | 1 - .../main/java/ch/dlcm/service/JobService.java | 3 + .../ch/dlcm/business/InfoPackageService.java | 2 + ...hivalInfoPackageRemoteResourceService.java | 5 + 11 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 DLCM-PreservationPlanning/src/main/java/ch/dlcm/job/ComplianceLevelUpdateJob.java diff --git a/DLCM-ArchivalStorage/src/main/java/ch/dlcm/controller/archivalstorage/AIPController.java b/DLCM-ArchivalStorage/src/main/java/ch/dlcm/controller/archivalstorage/AIPController.java index 89280e2b81..a1c312d06f 100644 --- a/DLCM-ArchivalStorage/src/main/java/ch/dlcm/controller/archivalstorage/AIPController.java +++ b/DLCM-ArchivalStorage/src/main/java/ch/dlcm/controller/archivalstorage/AIPController.java @@ -456,6 +456,12 @@ public class AIPController extends AbstractPackageController<AipDataFile, Archiv return this.changeAipStatus(PackageStatus.METADATA_UPGRADE_PENDING, aipId); } + @PostMapping(SolidifyConstants.URL_ID_PLUS_SEP + DLCMActionName.START_COMPLIANCE_LEVEL_UPDATE) + @TrustedUserPermissions + public HttpEntity<Result> updateComplianceLevel(@PathVariable("id") String aipId) { + return this.changeAipStatus(PackageStatus.COMPLIANCE_LEVEL_UPDATE_PENDING, aipId); + } + @Override protected <W extends RepresentationModel<W>> void addOthersLinks(W w) { w.add(Tool.filter(linkTo(this.getClass()).toUriComponentsBuilder(), DLCMConstants.ARCHIVAL_UNIT_PARAM, "true") diff --git a/DLCM-ArchivalStorage/src/main/java/ch/dlcm/service/AipStatusService.java b/DLCM-ArchivalStorage/src/main/java/ch/dlcm/service/AipStatusService.java index 0c7983b2b3..1e588b9471 100644 --- a/DLCM-ArchivalStorage/src/main/java/ch/dlcm/service/AipStatusService.java +++ b/DLCM-ArchivalStorage/src/main/java/ch/dlcm/service/AipStatusService.java @@ -172,6 +172,8 @@ public class AipStatusService extends AbstractAipService { case EDITING_METADATA -> this.updateMetadata(aip); case METADATA_UPGRADE_PENDING -> this.startMetadataUpgradeAip(aip); case UPGRADING_METADATA -> this.upgradeMetadata(aip); + case COMPLIANCE_LEVEL_UPDATE_PENDING -> this.startComplianceLevelUpdateAip(aip); + case UPDATING_COMPLIANCE_LEVEL -> this.updateComplianceLevel(aip); default -> throw new SolidifyRuntimeException( this.messageService.get("archival.aip.error.status", new Object[] { aip.getInfo().getStatus() })); @@ -390,6 +392,29 @@ public class AipStatusService extends AbstractAipService { } } + // AIP with COMPLIANCE_LEVEL_UPDATE_PENDING status + private void startComplianceLevelUpdateAip(ArchivalInfoPackage aip) { + // Start AIP metadata edition + aip.setPackageStatus(PackageStatus.UPDATING_COMPLIANCE_LEVEL); + this.logPackageMessage(LogLevel.INFO, aip, "compliance level updating is starting"); + } + + // AIP with UPDATING_COMPLIANCE_LEVEL status + private void updateComplianceLevel(ArchivalInfoPackage aip) { + try { + this.storageService.upgradeArchiveMetadata(aip); + this.updateAipWithNewPackage(aip); + aip.setPackageStatus(PackageStatus.REINDEXING); + aip.setLastArchiving(OffsetDateTime.now()); + } catch (IOException e) { + throw new SolidifyRuntimeException("IOException during AIP metadata update", e); + } catch (JAXBException e) { + throw new SolidifyRuntimeException("JAXBException during AIP metadata update", e); + } catch (NoSuchAlgorithmException | URISyntaxException e) { + throw new SolidifyRuntimeException("NoSuchAlgorithmException | URISyntaxException during AIP metadata update", e); + } + } + // AIP with PACKAGE_REPLICATION_PENDING status private void startReplicationPackageAip(ArchivalInfoPackage aip) { // Start AIP package replication diff --git a/DLCM-Common/src/main/resources/locale/messages_dlcm.properties b/DLCM-Common/src/main/resources/locale/messages_dlcm.properties index c279365837..46e2bc5633 100644 --- a/DLCM-Common/src/main/resources/locale/messages_dlcm.properties +++ b/DLCM-Common/src/main/resources/locale/messages_dlcm.properties @@ -361,6 +361,8 @@ package.status.fixing=Fixing package.status.fixity_error=Fixity error package.status.metadata_upgrade_pending=Metadata version upgrade pending package.status.upgrading_metadata=Upgrading metadata version to the latest +package.status.compliance_level_update_pending=Compliance level update pending +package.status.updating_compliance_level=Updating archive compliance level package.error.check_directory=Problem with package directory ({0}) in {1} ############# # Data file # diff --git a/DLCM-Model/src/main/java/ch/dlcm/model/PackageStatus.java b/DLCM-Model/src/main/java/ch/dlcm/model/PackageStatus.java index 4183e520e1..623bb8723f 100644 --- a/DLCM-Model/src/main/java/ch/dlcm/model/PackageStatus.java +++ b/DLCM-Model/src/main/java/ch/dlcm/model/PackageStatus.java @@ -71,6 +71,7 @@ public enum PackageStatus { FIX_PENDING, FIXING, METADATA_EDITION_PENDING, EDITING_METADATA, UPDATING_RETENTION, METADATA_UPGRADE_PENDING, UPGRADING_METADATA, + COMPLIANCE_LEVEL_UPDATE_PENDING, UPDATING_COMPLIANCE_LEVEL, REINDEXING, RELOADED, RESUBMITTING, @@ -146,6 +147,11 @@ public enum PackageStatus { || status == PackageStatus.UPGRADING_METADATA); } + public static boolean isComplianceUpdateProcess(PackageStatus status) { + return (status == PackageStatus.COMPLIANCE_LEVEL_UPDATE_PENDING + || status == PackageStatus.UPDATING_COMPLIANCE_LEVEL); + } + public static boolean isInProgress(PackageStatus status) { return (!isInError(status) && !isCompletedProcess(status) && !isDisposable(status)); } diff --git a/DLCM-Model/src/main/java/ch/dlcm/model/preservation/JobType.java b/DLCM-Model/src/main/java/ch/dlcm/model/preservation/JobType.java index f0edb676ee..b3798305af 100644 --- a/DLCM-Model/src/main/java/ch/dlcm/model/preservation/JobType.java +++ b/DLCM-Model/src/main/java/ch/dlcm/model/preservation/JobType.java @@ -36,6 +36,7 @@ public enum JobType { ARCHIVE_PRELOAD_SMALL("Archive [small size] Preloading", true, true), CHECK_COMPLIANCE_LEVEL("Check compliance level", true, true), CLEAN_SUBMISSION("Clean Submission", true, true), + COMPLIANCE_LEVEL_UPDATE("Compliance Level Update", false, true), DISPOSAL("Trigger Disposal Processes", true, true), FIXITY("Fixity", true, true), METADATA_MIGRATION("Metadata Version Migration", false, true), diff --git a/DLCM-Model/src/main/java/ch/dlcm/rest/DLCMActionName.java b/DLCM-Model/src/main/java/ch/dlcm/rest/DLCMActionName.java index ce5d27a22a..5999ab5200 100644 --- a/DLCM-Model/src/main/java/ch/dlcm/rest/DLCMActionName.java +++ b/DLCM-Model/src/main/java/ch/dlcm/rest/DLCMActionName.java @@ -86,6 +86,7 @@ public class DLCMActionName { public static final String PREPARE_DOWNLOAD = "prepare-download"; public static final String START_METADATA_EDITING = "start-metadata-editing"; public static final String START_METADATA_UPGRADE = "start-metadata-upgrade"; + public static final String START_COMPLIANCE_LEVEL_UPDATE = "start-compliance-level-update"; public static final String SET_APPROVED = "set-approved"; public static final String SET_REFUSED = "set-refused"; public static final String SET_PENDING = "set-pending"; diff --git a/DLCM-PreservationPlanning/src/main/java/ch/dlcm/job/ComplianceLevelUpdateJob.java b/DLCM-PreservationPlanning/src/main/java/ch/dlcm/job/ComplianceLevelUpdateJob.java new file mode 100644 index 0000000000..a083de7be1 --- /dev/null +++ b/DLCM-PreservationPlanning/src/main/java/ch/dlcm/job/ComplianceLevelUpdateJob.java @@ -0,0 +1,91 @@ +/*- + * %%---------------------------------------------------------------------------------------------- + * DLCM Technology - DLCM Preservation Planning - ComplianceLevelUpdateJob.java + * SPDX-License-Identifier: GPL-2.0-or-later + * %----------------------------------------------------------------------------------------------% + * Copyright (C) 2017 - 2022 University of Geneva + * %----------------------------------------------------------------------------------------------% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 2 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/gpl-2.0.html>. + * ----------------------------------------------------------------------------------------------%% + */ + +package ch.dlcm.job; + +import org.springframework.data.domain.Pageable; + +import ch.unige.solidify.rest.RestCollection; +import ch.unige.solidify.rest.Result; +import ch.unige.solidify.service.MessageService; + +import ch.dlcm.DLCMMetadataVersion; +import ch.dlcm.config.DLCMProperties; +import ch.dlcm.model.oais.ArchivalInfoPackage; +import ch.dlcm.model.preservation.PreservationJob; +import ch.dlcm.service.JobProcessingService; +import ch.dlcm.service.rest.trusted.TrustedArchivalInfoPackageRemoteResourceService; + +public class ComplianceLevelUpdateJob extends AbstractJob { + + private final TrustedArchivalInfoPackageRemoteResourceService aipRemoteResourceService; + + public ComplianceLevelUpdateJob( + DLCMProperties configService, + MessageService messageService, + JobProcessingService jobProcessingService, + TrustedArchivalInfoPackageRemoteResourceService aipRemoteResourceService) { + super(configService, messageService, jobProcessingService); + this.aipRemoteResourceService = aipRemoteResourceService; + } + + @Override + protected void execute(PreservationJob job, String executionId) { + final String storageUrl = this.configService.getDefaultArchivalStorageUrl(); + final long total = this.getTotal(storageUrl); + Pageable pageable = this.getPageRequest(); + RestCollection<ArchivalInfoPackage> collection; + do { + // List AIPs + collection = this.aipRemoteResourceService.getAipList(storageUrl, pageable); + pageable = pageable.next(); + for (final ArchivalInfoPackage aip : collection.getData()) { + try { + if (aip.getMetadataVersion() == DLCMMetadataVersion.getDefaultVersion()) { + job = this.addIgnoredItem(job, executionId, aip.getResId(), + this.getAipPublicUrl(storageUrl, aip.getResId()), + "No need to migrate metadata"); + } else { + // Upgrade AIP + final Result aipResult = this.aipRemoteResourceService.updateComplianceLevel(storageUrl, aip.getResId()); + job = this.logResult(aipResult, job, executionId, aip.getResId(), this.getAipPublicUrl(storageUrl, aip.getResId())); + + } + } catch (final Exception e) { + job = this.addInErrorItem(job, executionId, aip.getResId(), this.getAipPublicUrl(storageUrl, aip.getResId()), e.getMessage()); + } + // Save intermediate step by 10% + final long num = job.getLastExecution().getLastExecutionReport().getProcessedItems() + + job.getLastExecution().getLastExecutionReport().getIgnoredItems() + + job.getLastExecution().getLastExecutionReport().getInErrorItems(); + job = this.jobProcessingService.saveStep(job, executionId, num, total); + } + } while (collection.getPage().hasNext()); + } + + private long getTotal(String storageUrl) { + final RestCollection<ArchivalInfoPackage> collection = this.aipRemoteResourceService.getAipList(storageUrl, this.getPageRequestForTotal()); + return collection.getPage().getTotalItems(); + } + +} diff --git a/DLCM-PreservationPlanning/src/main/java/ch/dlcm/job/MetadataMigrationJob.java b/DLCM-PreservationPlanning/src/main/java/ch/dlcm/job/MetadataMigrationJob.java index 2a2aec99d3..dd25e0a981 100644 --- a/DLCM-PreservationPlanning/src/main/java/ch/dlcm/job/MetadataMigrationJob.java +++ b/DLCM-PreservationPlanning/src/main/java/ch/dlcm/job/MetadataMigrationJob.java @@ -71,7 +71,6 @@ public class MetadataMigrationJob extends AbstractJob { @Override protected void execute(PreservationJob job, String executionId) { - final long total = this.getTotal(this.defaultStorageUrl); Pageable pageable = this.getPageRequest(); RestCollection<ArchivalInfoPackage> collection; diff --git a/DLCM-PreservationPlanning/src/main/java/ch/dlcm/service/JobService.java b/DLCM-PreservationPlanning/src/main/java/ch/dlcm/service/JobService.java index 7b5e5b546c..582e3acd13 100644 --- a/DLCM-PreservationPlanning/src/main/java/ch/dlcm/service/JobService.java +++ b/DLCM-PreservationPlanning/src/main/java/ch/dlcm/service/JobService.java @@ -42,6 +42,7 @@ import ch.dlcm.job.ApplyAipActionJob; import ch.dlcm.job.CheckComplianceLevelJob; import ch.dlcm.job.CheckReplicationJob; import ch.dlcm.job.CleanSubmissionJob; +import ch.dlcm.job.ComplianceLevelUpdateJob; import ch.dlcm.job.DisposalJob; import ch.dlcm.job.MetadataMigrationJob; import ch.dlcm.job.PreloadAipJob; @@ -133,6 +134,8 @@ public class JobService extends DLCMService { AbstractJob jobProcess = switch (job.getJobType()) { case METADATA_MIGRATION -> new MetadataMigrationJob(this.dlcmProperties, this.messageService, this.jobProcessingService, this.depositRemoteResourceService, this.sipRemoteResourceService, this.aipRemoteResourceService); + case COMPLIANCE_LEVEL_UPDATE -> new ComplianceLevelUpdateJob(this.dlcmProperties, this.messageService, this.jobProcessingService, + this.aipRemoteResourceService); case RELOAD -> new ReloadAipJob(this.dlcmProperties, this.messageService, this.jobProcessingService, this.aipRemoteResourceService); case REPLICATION -> new ReplicationJob(this.dlcmProperties, this.messageService, this.jobProcessingService, this.replicationService, this.archiveMetadataRemoteResourceService); diff --git a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/business/InfoPackageService.java b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/business/InfoPackageService.java index 2ff30e459f..420ad8ab4a 100644 --- a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/business/InfoPackageService.java +++ b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/business/InfoPackageService.java @@ -102,6 +102,8 @@ public abstract class InfoPackageService<T extends Resource> extends CompositeRe return PackageStatus.TOMBSTONE_REPLICATION_PENDING; } else if (PackageStatus.isMetadataUpgradeProcess(statusBeforeError)) { return PackageStatus.METADATA_UPGRADE_PENDING; + } else if (PackageStatus.isComplianceUpdateProcess(statusBeforeError)) { + return PackageStatus.COMPLIANCE_LEVEL_UPDATE_PENDING; } else { return PackageStatus.IN_PREPARATION; } diff --git a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/rest/abstractservice/ArchivalInfoPackageRemoteResourceService.java b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/rest/abstractservice/ArchivalInfoPackageRemoteResourceService.java index 29f8ebe1c1..014b0c164d 100644 --- a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/rest/abstractservice/ArchivalInfoPackageRemoteResourceService.java +++ b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/rest/abstractservice/ArchivalInfoPackageRemoteResourceService.java @@ -303,6 +303,11 @@ public abstract class ArchivalInfoPackageRemoteResourceService extends RemoteRes return this.restClientService.postResourceAction(url); } + public Result updateComplianceLevel(String storageUrl, String aipId) { + String url = this.getArchivalStorageUrlWithId(storageUrl, aipId) + "/" + DLCMActionName.START_COMPLIANCE_LEVEL_UPDATE; + return this.restClientService.postResourceAction(url); + } + public RestCollection<ArchivalInfoPackage> getAipList(String storageUrl, Pageable pageable) { String jsonString = this.restClientService.getResource(storageUrl + "/" + ResourceName.AIP, pageable); return new RestCollection<>(jsonString, ArchivalInfoPackage.class); -- GitLab From 9e83d40ecb084089345adf1456ee4df67666d750 Mon Sep 17 00:00:00 2001 From: Hugues <hugues.cazeaux@unige.ch> Date: Wed, 5 Feb 2025 23:54:25 +0100 Subject: [PATCH 10/18] chore: reformat XSLT --- .../resources/xslt/dlcm4migration-1.0-1.1.xsl | 8 ++++++-- .../resources/xslt/dlcm4migration-1.1-2.0.xsl | 8 ++++++-- .../resources/xslt/dlcm4migration-2.0-2.1.xsl | 20 +++++++------------ .../resources/xslt/dlcm4migration-2.1-3.0.xsl | 8 ++++++-- .../resources/xslt/dlcm4migration-3.0-3.1.xsl | 4 +++- .../resources/xslt/dlcm4migration-3.1-4.0.xsl | 8 ++++++-- 6 files changed, 34 insertions(+), 22 deletions(-) diff --git a/DLCM-Model/src/main/resources/xslt/dlcm4migration-1.0-1.1.xsl b/DLCM-Model/src/main/resources/xslt/dlcm4migration-1.0-1.1.xsl index cd37d0fdbb..c2cbb879b2 100644 --- a/DLCM-Model/src/main/resources/xslt/dlcm4migration-1.0-1.1.xsl +++ b/DLCM-Model/src/main/resources/xslt/dlcm4migration-1.0-1.1.xsl @@ -14,12 +14,16 @@ <!-- METS PROFILE --> <xsl:template match="mets:mets/@PROFILE"> - <xsl:attribute name="PROFILE"><xsl:text>dlcm_profile-1.1.xml</xsl:text></xsl:attribute> + <xsl:attribute name="PROFILE"> + <xsl:text>dlcm_profile-1.1.xml</xsl:text> + </xsl:attribute> </xsl:template> <!-- METS MDTYPEVERSION --> <xsl:template match="mets:mets/mets:dmdSec/mets:mdWrap/@MDTYPEVERSION"> - <xsl:attribute name="MDTYPEVERSION"><xsl:text>DataCite Metadata Schema 4.0 (dlcm_datacite-1.0.xsd)</xsl:text></xsl:attribute> + <xsl:attribute name="MDTYPEVERSION"> + <xsl:text>DataCite Metadata Schema 4.0 (dlcm_datacite-1.0.xsd)</xsl:text> + </xsl:attribute> </xsl:template> </xsl:stylesheet> diff --git a/DLCM-Model/src/main/resources/xslt/dlcm4migration-1.1-2.0.xsl b/DLCM-Model/src/main/resources/xslt/dlcm4migration-1.1-2.0.xsl index 0a079b79d5..4abac31f7f 100644 --- a/DLCM-Model/src/main/resources/xslt/dlcm4migration-1.1-2.0.xsl +++ b/DLCM-Model/src/main/resources/xslt/dlcm4migration-1.1-2.0.xsl @@ -53,12 +53,16 @@ <!-- Migrate METS Profile --> <xsl:template match="mets:mets/@PROFILE"> - <xsl:attribute name="PROFILE"><xsl:text>dlcm_profile-2.0.xml</xsl:text></xsl:attribute> + <xsl:attribute name="PROFILE" + ><xsl:text>dlcm_profile-2.0.xml</xsl:text> + </xsl:attribute> </xsl:template> <!-- Migrate METS MDTYPEVERSION --> <xsl:template match="mets:mets/mets:dmdSec/mets:mdWrap/@MDTYPEVERSION"> - <xsl:attribute name="MDTYPEVERSION"><xsl:text>DataCite Metadata Schema 4.3 (dlcm_datacite-2.0.xsd)</xsl:text></xsl:attribute> + <xsl:attribute name="MDTYPEVERSION"> + <xsl:text>DataCite Metadata Schema 4.3 (dlcm_datacite-2.0.xsd)</xsl:text> + </xsl:attribute> </xsl:template> <!-- Add DataCite 'nameType' attribute for each 'datacite:creatorName' --> diff --git a/DLCM-Model/src/main/resources/xslt/dlcm4migration-2.0-2.1.xsl b/DLCM-Model/src/main/resources/xslt/dlcm4migration-2.0-2.1.xsl index 48090c72a2..ab1abb1252 100644 --- a/DLCM-Model/src/main/resources/xslt/dlcm4migration-2.0-2.1.xsl +++ b/DLCM-Model/src/main/resources/xslt/dlcm4migration-2.0-2.1.xsl @@ -1,7 +1,7 @@ <?xml version="1.0" encoding="UTF-8"?> <xsl:stylesheet version="2.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:mets="http://www.loc.gov/METS/" xmlns:datacite="http://datacite.org/schema/kernel-4" xmlns:premis="http://www.loc.gov/premis/v3" - xmlns:dlcm="http://www.dlcm.ch/dlcm/v2" xmlns:fits="http://hul.harvard.edu/ois/xml/ns/fits/fits_output" exclude-result-prefixes="datacite"> + xmlns:dlcm="http://www.dlcm.ch/dlcm/v2" xmlns:fits="http://hul.harvard.edu/ois/xml/ns/fits/fits_output"> <xsl:output method="xml" omit-xml-declaration="no" indent="yes" encoding="UTF-8" /> @@ -14,22 +14,16 @@ <!-- METS Profile --> <xsl:template match="mets:mets/@PROFILE"> - <xsl:attribute name="PROFILE"><xsl:text>dlcm_profile-2.1.xml</xsl:text></xsl:attribute> + <xsl:attribute name="PROFILE"> + <xsl:text>dlcm_profile-2.1.xml</xsl:text> + </xsl:attribute> </xsl:template> <!-- METS MDTYPEVERSION --> <xsl:template match="mets:mets/mets:dmdSec/mets:mdWrap/@MDTYPEVERSION"> - <xsl:attribute name="MDTYPEVERSION"><xsl:text>DataCite Metadata Schema 4.3 (dlcm_datacite-2.0.xsd)</xsl:text></xsl:attribute> - </xsl:template> - - <!-- METS Profile --> - <xsl:template match="mets:mets/@PROFILE"> - <xsl:attribute name="PROFILE"><xsl:text>dlcm_profile-2.1.xml</xsl:text></xsl:attribute> - </xsl:template> - - <!-- METS MDTYPEVERSION --> - <xsl:template match="mets:mets/mets:dmdSec/mets:mdWrap/@MDTYPEVERSION"> - <xsl:attribute name="MDTYPEVERSION"><xsl:text>DataCite Metadata Schema 4.3 (dlcm_datacite-2.0.xsd)</xsl:text></xsl:attribute> + <xsl:attribute name="MDTYPEVERSION"> + <xsl:text>DataCite Metadata Schema 4.3 (dlcm_datacite-2.0.xsd)</xsl:text> + </xsl:attribute> </xsl:template> </xsl:stylesheet> diff --git a/DLCM-Model/src/main/resources/xslt/dlcm4migration-2.1-3.0.xsl b/DLCM-Model/src/main/resources/xslt/dlcm4migration-2.1-3.0.xsl index c3330ebb15..e5f003ab33 100644 --- a/DLCM-Model/src/main/resources/xslt/dlcm4migration-2.1-3.0.xsl +++ b/DLCM-Model/src/main/resources/xslt/dlcm4migration-2.1-3.0.xsl @@ -14,12 +14,16 @@ <!-- METS Profile --> <xsl:template match="mets:mets/@PROFILE"> - <xsl:attribute name="PROFILE"><xsl:text>dlcm_profile-3.0.xml</xsl:text></xsl:attribute> + <xsl:attribute name="PROFILE"> + <xsl:text>dlcm_profile-3.0.xml</xsl:text> + </xsl:attribute> </xsl:template> <!-- METS MDTYPEVERSION --> <xsl:template match="mets:mets/mets:dmdSec/mets:mdWrap/@MDTYPEVERSION"> - <xsl:attribute name="MDTYPEVERSION"><xsl:text>DataCite Metadata Schema 4.4 (dlcm_datacite-3.0.xsd)</xsl:text></xsl:attribute> + <xsl:attribute name="MDTYPEVERSION"> + <xsl:text>DataCite Metadata Schema 4.4 (dlcm_datacite-3.0.xsd)</xsl:text> + </xsl:attribute> </xsl:template> </xsl:stylesheet> diff --git a/DLCM-Model/src/main/resources/xslt/dlcm4migration-3.0-3.1.xsl b/DLCM-Model/src/main/resources/xslt/dlcm4migration-3.0-3.1.xsl index 359a7ea7c2..3d308a727e 100644 --- a/DLCM-Model/src/main/resources/xslt/dlcm4migration-3.0-3.1.xsl +++ b/DLCM-Model/src/main/resources/xslt/dlcm4migration-3.0-3.1.xsl @@ -14,7 +14,9 @@ <!-- METS Profile --> <xsl:template match="mets:mets/@PROFILE"> - <xsl:attribute name="PROFILE"><xsl:text>dlcm_profile-3.1.xml</xsl:text></xsl:attribute> + <xsl:attribute name="PROFILE"> + <xsl:text>dlcm_profile-3.1.xml</xsl:text> + </xsl:attribute> </xsl:template> </xsl:stylesheet> diff --git a/DLCM-Model/src/main/resources/xslt/dlcm4migration-3.1-4.0.xsl b/DLCM-Model/src/main/resources/xslt/dlcm4migration-3.1-4.0.xsl index 7b4a75988c..09a2a79568 100644 --- a/DLCM-Model/src/main/resources/xslt/dlcm4migration-3.1-4.0.xsl +++ b/DLCM-Model/src/main/resources/xslt/dlcm4migration-3.1-4.0.xsl @@ -53,12 +53,16 @@ <!-- Migrate METS Profile --> <xsl:template match="mets:mets/@PROFILE"> - <xsl:attribute name="PROFILE"><xsl:text>dlcm_profile-4.0.xml</xsl:text></xsl:attribute> + <xsl:attribute name="PROFILE"> + <xsl:text>dlcm_profile-4.0.xml</xsl:text> + </xsl:attribute> </xsl:template> <!-- Migrate METS MDTYPEVERSION --> <xsl:template match="mets:mets/mets:dmdSec/mets:mdWrap/@MDTYPEVERSION"> - <xsl:attribute name="MDTYPEVERSION"><xsl:text>DataCite Metadata Schema 4.5 (dlcm_datacite-4.0.xsd)</xsl:text></xsl:attribute> + <xsl:attribute name="MDTYPEVERSION"> + <xsl:text>DataCite Metadata Schema 4.5 (dlcm_datacite-4.0.xsd)</xsl:text> + </xsl:attribute> </xsl:template> </xsl:stylesheet> -- GitLab From 4d14fe4684c8a427d8b4052db8a40bb419e40fee Mon Sep 17 00:00:00 2001 From: Hugues <hugues.cazeaux@unige.ch> Date: Thu, 6 Feb 2025 00:46:32 +0100 Subject: [PATCH 11/18] feat: compliance level update --- .../archivalstorage/AIPController.java | 6 ++- .../ch/dlcm/service/AipStatusService.java | 15 +++--- .../java/ch/dlcm/storage/StorageService.java | 43 ++++++++++++++--- .../ch/dlcm/job/ComplianceLevelUpdateJob.java | 16 +++---- .../java/ch/dlcm/service/MetadataService.java | 6 +++ .../service/metadata/MetadataGenerator.java | 2 + .../service/metadata/MetadataVersion1.java | 5 ++ .../service/metadata/MetadataVersion2.java | 5 ++ .../service/metadata/MetadataVersion3.java | 5 ++ .../service/metadata/MetadataVersion4.java | 48 +++++++++++++++++-- 10 files changed, 125 insertions(+), 26 deletions(-) diff --git a/DLCM-ArchivalStorage/src/main/java/ch/dlcm/controller/archivalstorage/AIPController.java b/DLCM-ArchivalStorage/src/main/java/ch/dlcm/controller/archivalstorage/AIPController.java index a1c312d06f..ece1673f67 100644 --- a/DLCM-ArchivalStorage/src/main/java/ch/dlcm/controller/archivalstorage/AIPController.java +++ b/DLCM-ArchivalStorage/src/main/java/ch/dlcm/controller/archivalstorage/AIPController.java @@ -556,12 +556,14 @@ public class AIPController extends AbstractPackageController<AipDataFile, Archiv final Result result = new Result(aipId); if (currentStatus == PackageStatus.COMPLETED && (newStatus == PackageStatus.METADATA_EDITION_PENDING - || newStatus == PackageStatus.METADATA_UPGRADE_PENDING)) { + || newStatus == PackageStatus.METADATA_UPGRADE_PENDING + || newStatus == PackageStatus.COMPLIANCE_LEVEL_UPDATE_PENDING)) { aip.getInfo().setStatus(newStatus); aip = this.itemService.save(aip); result.setStatus(Result.ActionStatus.EXECUTED); result.setMesssage(this.messageService.get("message.aip.status")); - if (newStatus == PackageStatus.METADATA_UPGRADE_PENDING) { + if (newStatus == PackageStatus.METADATA_UPGRADE_PENDING + || newStatus == PackageStatus.COMPLIANCE_LEVEL_UPDATE_PENDING) { ((ArchivalInfoPackageService) this.itemService).putPackageInProcessingQueue(aip); } return new ResponseEntity<>(result, HttpStatus.OK); diff --git a/DLCM-ArchivalStorage/src/main/java/ch/dlcm/service/AipStatusService.java b/DLCM-ArchivalStorage/src/main/java/ch/dlcm/service/AipStatusService.java index 1e588b9471..7471cf51f5 100644 --- a/DLCM-ArchivalStorage/src/main/java/ch/dlcm/service/AipStatusService.java +++ b/DLCM-ArchivalStorage/src/main/java/ch/dlcm/service/AipStatusService.java @@ -402,14 +402,17 @@ public class AipStatusService extends AbstractAipService { // AIP with UPDATING_COMPLIANCE_LEVEL status private void updateComplianceLevel(ArchivalInfoPackage aip) { try { - this.storageService.upgradeArchiveMetadata(aip); - this.updateAipWithNewPackage(aip); - aip.setPackageStatus(PackageStatus.REINDEXING); - aip.setLastArchiving(OffsetDateTime.now()); + if (this.storageService.updateComplianceLevel(aip)) { + this.updateAipWithNewPackage(aip); + aip.setPackageStatus(PackageStatus.REINDEXING); + aip.setLastArchiving(OffsetDateTime.now()); + } else { + this.completeAip(aip); + } } catch (IOException e) { - throw new SolidifyRuntimeException("IOException during AIP metadata update", e); + throw new SolidifyRuntimeException("IOException during AIP compliance level update", e); } catch (JAXBException e) { - throw new SolidifyRuntimeException("JAXBException during AIP metadata update", e); + throw new SolidifyRuntimeException("JAXBException during AIP compliance level update", e); } catch (NoSuchAlgorithmException | URISyntaxException e) { throw new SolidifyRuntimeException("NoSuchAlgorithmException | URISyntaxException during AIP metadata update", e); } diff --git a/DLCM-ArchivalStorage/src/main/java/ch/dlcm/storage/StorageService.java b/DLCM-ArchivalStorage/src/main/java/ch/dlcm/storage/StorageService.java index f30cad1cf5..2e8003be05 100644 --- a/DLCM-ArchivalStorage/src/main/java/ch/dlcm/storage/StorageService.java +++ b/DLCM-ArchivalStorage/src/main/java/ch/dlcm/storage/StorageService.java @@ -54,6 +54,7 @@ import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; import ch.unige.solidify.ChecksumAlgorithm; +import ch.unige.solidify.SolidifyConstants; import ch.unige.solidify.exception.SolidifyCheckingException; import ch.unige.solidify.exception.SolidifyProcessingException; import ch.unige.solidify.service.MessageService; @@ -248,8 +249,6 @@ public abstract class StorageService extends DLCMService { // Check & extract archive final Path aipPackageFolder = Paths.get(this.checkPackage(aip, false, false)); final Path aipDataFolder = this.getDataFolder(aip.getArchiveContainer(), aipPackageFolder); - // Get updated folder - final Path aipUpdateFolder = this.getAIPFolder(aip.getResId()).resolve(DLCMConstants.UPDATE_FOLDER); // Migrate metadata MetadataMigrationTool.migrateMetadataToDefaultVersion(aip.getInfo().getMetadataVersion(), aipDataFolder.resolve(Package.METADATA_FILE.getName())); @@ -262,10 +261,6 @@ public abstract class StorageService extends DLCMService { if (FileTool.checkFile(aipPackageFolder) && !FileTool.deleteFolder(aipPackageFolder)) { throw new DLCMPreservationException(this.messageService.get(AIP_ERROR_PURGE_FILES_MSG)); } - // Purge update folder (temporary folder for archive files) - if (FileTool.checkFile(aipUpdateFolder) && !FileTool.deleteFolder(aipUpdateFolder)) { - throw new DLCMPreservationException(this.messageService.get(AIP_ERROR_PURGE_FILES_MSG)); - } // Set the archive size aip.setArchiveSize(FileTool.getSize(newPackageFile)); aip.setArchiveFileNumber(aip.getArchiveFileNumber() + 1); // add one to ArchiveFileNumber as we create xml file every time we do an update @@ -275,6 +270,42 @@ public abstract class StorageService extends DLCMService { aip.incrementUpdateNumber(); } + // Update compliance level + public boolean updateComplianceLevel(ArchivalInfoPackage aip) throws IOException, JAXBException, URISyntaxException { + // Check & extract archive + final Path aipPackageFolder = Paths.get(this.checkPackage(aip, false, false)); + final Path aipDataFolder = this.getDataFolder(aip.getArchiveContainer(), aipPackageFolder); + final Path metadataXml = aipDataFolder.resolve(Package.METADATA_FILE.getName()); + // Copy original metadata XML file + final Path originalXml = metadataXml.getParent() + .resolve(metadataXml.getFileName().toString() + .replaceFirst(SolidifyConstants.XML_EXT, + "-" + OffsetDateTime.now().format(DateTimeFormatter.ofPattern(StringTool.DATE_TIME_FORMAT_FOR_FILE)) + + SolidifyConstants.XML_EXT)); + if (!FileTool.copyFile(metadataXml, originalXml)) { + throw new SolidifyProcessingException("Cannot copy the original metadata file: " + metadataXml + " => " + originalXml); + } + // Upgrade metadata by adding Premis event + final boolean updated = this.metadataService.updateComplianceLevel(aip, + aipPackageFolder.resolve(this.getMetadataRelativePath(aip.getArchiveContainer()))); + if (updated) { + // Generate new package + final Path newPackageFile = this.generateNewPackage(aip, aipDataFolder); + // Set the archive size + aip.setArchiveSize(FileTool.getSize(newPackageFile)); + aip.setArchiveFileNumber(aip.getArchiveFileNumber() + 1); // add one to ArchiveFileNumber as we create xml file every time we do an update + // Save new archive + this.replaceAIP(aip, newPackageFile); + // Add update + aip.incrementUpdateNumber(); + } + // Purge package folder (temporary folder for updating archive ) + if (FileTool.checkFile(aipPackageFolder) && !FileTool.deleteFolder(aipPackageFolder)) { + throw new DLCMPreservationException(this.messageService.get(AIP_ERROR_PURGE_FILES_MSG)); + } + return updated; + } + // ********************* // ** Default Methods ** // ********************* diff --git a/DLCM-PreservationPlanning/src/main/java/ch/dlcm/job/ComplianceLevelUpdateJob.java b/DLCM-PreservationPlanning/src/main/java/ch/dlcm/job/ComplianceLevelUpdateJob.java index a083de7be1..b6913662a0 100644 --- a/DLCM-PreservationPlanning/src/main/java/ch/dlcm/job/ComplianceLevelUpdateJob.java +++ b/DLCM-PreservationPlanning/src/main/java/ch/dlcm/job/ComplianceLevelUpdateJob.java @@ -29,8 +29,8 @@ import ch.unige.solidify.rest.RestCollection; import ch.unige.solidify.rest.Result; import ch.unige.solidify.service.MessageService; -import ch.dlcm.DLCMMetadataVersion; import ch.dlcm.config.DLCMProperties; +import ch.dlcm.model.PackageStatus; import ch.dlcm.model.oais.ArchivalInfoPackage; import ch.dlcm.model.preservation.PreservationJob; import ch.dlcm.service.JobProcessingService; @@ -61,15 +61,14 @@ public class ComplianceLevelUpdateJob extends AbstractJob { pageable = pageable.next(); for (final ArchivalInfoPackage aip : collection.getData()) { try { - if (aip.getMetadataVersion() == DLCMMetadataVersion.getDefaultVersion()) { - job = this.addIgnoredItem(job, executionId, aip.getResId(), - this.getAipPublicUrl(storageUrl, aip.getResId()), - "No need to migrate metadata"); - } else { - // Upgrade AIP + if (aip.getInfo().getStatus() == PackageStatus.COMPLETED) { + // Update compliance level final Result aipResult = this.aipRemoteResourceService.updateComplianceLevel(storageUrl, aip.getResId()); job = this.logResult(aipResult, job, executionId, aip.getResId(), this.getAipPublicUrl(storageUrl, aip.getResId())); - + } else { + job = this.addIgnoredItem(job, executionId, aip.getResId(), + this.getAipPublicUrl(storageUrl, aip.getResId()), + "Archive not ready for compliance level update"); } } catch (final Exception e) { job = this.addInErrorItem(job, executionId, aip.getResId(), this.getAipPublicUrl(storageUrl, aip.getResId()), e.getMessage()); @@ -87,5 +86,4 @@ public class ComplianceLevelUpdateJob extends AbstractJob { final RestCollection<ArchivalInfoPackage> collection = this.aipRemoteResourceService.getAipList(storageUrl, this.getPageRequestForTotal()); return collection.getPage().getTotalItems(); } - } diff --git a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/MetadataService.java b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/MetadataService.java index 8400e62335..a248731254 100644 --- a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/MetadataService.java +++ b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/MetadataService.java @@ -437,6 +437,12 @@ public class MetadataService extends DLCMService implements OAIMetadataService { metadataGenerator.upgradeMetadata(aip, metadataFile); } + public boolean updateComplianceLevel(ArchivalInfoPackage aip, Path metadataFile) throws JAXBException, IOException { + // Get metadata generator for the version + final MetadataGenerator metadataGenerator = this.getMetadataGenerator(aip.getInfo().getMetadataVersion()); + return metadataGenerator.updateCompliance(aip, metadataFile); + } + // ** Disposal ** public void updateMetadataForDisposal(ArchivalInfoPackage aip, Path metadataFile) throws JAXBException, IOException { diff --git a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataGenerator.java b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataGenerator.java index 86a5866fa1..a221d3f9c2 100644 --- a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataGenerator.java +++ b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataGenerator.java @@ -262,6 +262,8 @@ public abstract class MetadataGenerator { public abstract void upgradeMetadata(ArchivalInfoPackage aip, Path metadataFile) throws IOException, JAXBException; + public abstract boolean updateCompliance(ArchivalInfoPackage aip, Path metadataFile) throws IOException, JAXBException; + public abstract Map<String, String> extractInfoFromMetadata(Path metadataFile) throws IOException, JAXBException; public abstract void extractDataciteMetadataFromFile(Path filePath, OutputStream outputStream) throws JAXBException, IOException; diff --git a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion1.java b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion1.java index 287412a52e..500418e35a 100644 --- a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion1.java +++ b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion1.java @@ -1964,6 +1964,11 @@ public class MetadataVersion1 extends MetadataGenerator { throw new UnsupportedOperationException("Not supported for this metadata version"); } + @Override + public boolean updateCompliance(ArchivalInfoPackage aip, Path metadataFile) throws IOException, JAXBException { + return false; + } + private void updatePreservationPolicy(ArchivalInfoPackage aip, IntellectualEntity premisAip) { // Remove duplicate if (premisAip.getPreservationLevel().size() > 1) { diff --git a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion2.java b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion2.java index 0f5b9f0b30..082e3e4125 100644 --- a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion2.java +++ b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion2.java @@ -536,6 +536,11 @@ public class MetadataVersion2 extends MetadataGenerator { throw new UnsupportedOperationException("Not supported for this metadata version"); } + @Override + public boolean updateCompliance(ArchivalInfoPackage aip, Path metadataFile) throws IOException, JAXBException { + return false; + } + @Override public Map<String, String> extractInfoFromMetadata(Path metadataFile) throws IOException, JAXBException { final Map<String, String> infoList = new HashMap<>(); diff --git a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion3.java b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion3.java index ab5b2d1ddb..19adc55a9b 100644 --- a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion3.java +++ b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion3.java @@ -551,6 +551,11 @@ public class MetadataVersion3 extends MetadataGenerator { throw new UnsupportedOperationException("Not supported for this metadata version"); } + @Override + public boolean updateCompliance(ArchivalInfoPackage aip, Path metadataFile) throws IOException, JAXBException { + return false; + } + @Override public Map<String, String> extractInfoFromMetadata(Path metadataFile) throws IOException, JAXBException { final Map<String, String> infoList = new HashMap<>(); diff --git a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion4.java b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion4.java index 3dfd90b119..ed365d4c98 100644 --- a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion4.java +++ b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion4.java @@ -570,7 +570,7 @@ public class MetadataVersion4 extends MetadataGenerator { // Load Mets metadata from file final Mets mets = this.loadMetadataFile(metadataFile); // Get Premis object - PremisComplexType premis = mets.getAmdSec().getDigiprovMD().getMdWrap().getXmlData().getPremis(); + final PremisComplexType premis = mets.getAmdSec().getDigiprovMD().getMdWrap().getXmlData().getPremis(); final IntellectualEntity premisAip = this.findObject(premis.getObject(), ResourceName.AIP); // add history event this.addHistoryEvents(ResourceName.AIP, aip.getResId(), premis.getEvent(), premisAip.getObjectIdentifier()); @@ -586,32 +586,74 @@ public class MetadataVersion4 extends MetadataGenerator { this.saveXmlInFile(mets, metadataFile.toString()); } - private void updateComplianceLevel(PremisComplexType premis) throws JAXBException { + @Override + public boolean updateCompliance(ArchivalInfoPackage aip, Path metadataFile) throws IOException, JAXBException { + // Load Mets metadata from file + final Mets mets = this.loadMetadataFile(metadataFile); + // Get Premis object + final PremisComplexType premis = mets.getAmdSec().getDigiprovMD().getMdWrap().getXmlData().getPremis(); + // Update compliance level + if (!this.updateComplianceLevel(premis)) { + return false; + } + final IntellectualEntity premisAip = this.findObject(premis.getObject(), ResourceName.AIP); + // add history event + this.addHistoryEvents(ResourceName.AIP, aip.getResId(), premis.getEvent(), premisAip.getObjectIdentifier()); + // Update PREMIS metadata checksum + this.updateChecksum(mets.getAmdSec().getDigiprovMD().getMdWrap()); + // Update PREMIS objects for collection + if (aip.isCollection()) { + this.updatePremisObjectsForCollection(new MdWrapWrapper(mets.getAmdSec().getDigiprovMD().getMdWrap()), aip); + } + // Save in file + this.saveXmlInFile(mets, metadataFile.toString()); + return true; + } + + private boolean updateComplianceLevel(PremisComplexType premis) throws JAXBException { + boolean updated = false; + // Get deposit ID final IntellectualEntity premisDeposit = this.findObject(premis.getObject(), ResourceName.DEPOSIT); + // Create Deposit object final Deposit deposit = new Deposit(); deposit.setResId(premisDeposit.getObjectIdentifier().get(0).getObjectIdentifierValue()); + // For each data file for (ObjectComplexType objectComplexType : premis.getObject().stream().filter(PremisFile.class::isInstance).toList()) { final PremisFile premisFile = (PremisFile) objectComplexType; Optional<ObjectIdentifierComplexType> fileId = this.getPremisObjectIdentifier(premisFile, this.getPackageType(ResourceName.DATAFILE)); if (fileId.isPresent()) { + // Create Deposit Data File object final DepositDataFile depositDataFile = new DepositDataFile(); + // Data File ID depositDataFile.setResId(fileId.get().getObjectIdentifierValue()); + // Data File deposit depositDataFile.setInfoPackage(deposit); + // Data File size depositDataFile.setFileSize(premisFile.getObjectCharacteristics().get(0).getSize()); + // Get FITS & DLCM info final Optional<ExtensionComplexType> fitsExtension = this.getPremisObjectExtension(premisFile, Fits.class); final Optional<ExtensionComplexType> dlcmExtension = this.getPremisObjectExtension(premisFile, DlcmInfo.class); if (fitsExtension.isPresent() && dlcmExtension.isPresent()) { final Fits fitsInfo = (Fits) fitsExtension.get().getAny().get(0); final DlcmInfo dlcmInfo = (DlcmInfo) dlcmExtension.get().getAny().get(0); + // Data File File Format info (FITS) depositDataFile.getFileFormat().setDetails(this.getFitsInfo(fitsInfo)); - dlcmInfo.setComplianceLevel(this.fileFormatService.assessComplianceLevel(depositDataFile.getResId(), depositDataFile).value()); + final int originalComplianceLevel = dlcmInfo.getComplianceLevel(); + final int newComplianceLevel = this.fileFormatService.assessComplianceLevel(depositDataFile.getResId(), depositDataFile).value(); + // Change compliance level + if (newComplianceLevel != originalComplianceLevel) { + updated = true; + dlcmInfo.setComplianceLevel(newComplianceLevel); + } if (dlcmInfo.getDataClassification().getValue().equals("Unknown")) { + updated = true; final DepositDataFile df = this.depositRemoteService.getDepositDataFile(deposit.getResId(), depositDataFile.getResId()); dlcmInfo.setDataClassification(this.createDataClassification(df)); } } } } + return updated; } private PremisComplexType addPremisInfoFromUpdatedFiles(Mets mets, PremisComplexType premis, -- GitLab From 063a4e0fc6506075e2a1658ffc42d850bb6cf1c2 Mon Sep 17 00:00:00 2001 From: Hugues <hugues.cazeaux@unige.ch> Date: Thu, 6 Feb 2025 12:40:03 +0100 Subject: [PATCH 12/18] chore: fixed rebase --- .../ch/dlcm/job/ComplianceLevelUpdateJob.java | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/DLCM-PreservationPlanning/src/main/java/ch/dlcm/job/ComplianceLevelUpdateJob.java b/DLCM-PreservationPlanning/src/main/java/ch/dlcm/job/ComplianceLevelUpdateJob.java index b6913662a0..f7bd530e69 100644 --- a/DLCM-PreservationPlanning/src/main/java/ch/dlcm/job/ComplianceLevelUpdateJob.java +++ b/DLCM-PreservationPlanning/src/main/java/ch/dlcm/job/ComplianceLevelUpdateJob.java @@ -40,38 +40,41 @@ public class ComplianceLevelUpdateJob extends AbstractJob { private final TrustedArchivalInfoPackageRemoteResourceService aipRemoteResourceService; + private final String defaultStorageUrl; + public ComplianceLevelUpdateJob( - DLCMProperties configService, + DLCMProperties dlcmProperties, MessageService messageService, JobProcessingService jobProcessingService, TrustedArchivalInfoPackageRemoteResourceService aipRemoteResourceService) { - super(configService, messageService, jobProcessingService); + super(dlcmProperties, messageService, jobProcessingService); this.aipRemoteResourceService = aipRemoteResourceService; + this.defaultStorageUrl = dlcmProperties.getDefaultArchivalStorageUrl(); } @Override protected void execute(PreservationJob job, String executionId) { - final String storageUrl = this.configService.getDefaultArchivalStorageUrl(); - final long total = this.getTotal(storageUrl); + final long total = this.getTotal(this.defaultStorageUrl); Pageable pageable = this.getPageRequest(); RestCollection<ArchivalInfoPackage> collection; do { // List AIPs - collection = this.aipRemoteResourceService.getAipList(storageUrl, pageable); + collection = this.aipRemoteResourceService.getAipList(this.defaultStorageUrl, pageable); pageable = pageable.next(); for (final ArchivalInfoPackage aip : collection.getData()) { try { if (aip.getInfo().getStatus() == PackageStatus.COMPLETED) { // Update compliance level - final Result aipResult = this.aipRemoteResourceService.updateComplianceLevel(storageUrl, aip.getResId()); - job = this.logResult(aipResult, job, executionId, aip.getResId(), this.getAipPublicUrl(storageUrl, aip.getResId())); + final Result aipResult = this.aipRemoteResourceService.updateComplianceLevel(this.defaultStorageUrl, aip.getResId()); + job = this.logResult(aipResult, job, executionId, aip.getResId(), this.getAipPublicUrl(this.defaultStorageUrl, aip.getResId())); } else { job = this.addIgnoredItem(job, executionId, aip.getResId(), - this.getAipPublicUrl(storageUrl, aip.getResId()), + this.getAipPublicUrl(this.defaultStorageUrl, aip.getResId()), "Archive not ready for compliance level update"); } } catch (final Exception e) { - job = this.addInErrorItem(job, executionId, aip.getResId(), this.getAipPublicUrl(storageUrl, aip.getResId()), e.getMessage()); + job = this.addInErrorItem(job, executionId, aip.getResId(), this.getAipPublicUrl(this.defaultStorageUrl, aip.getResId()), + e.getMessage()); } // Save intermediate step by 10% final long num = job.getLastExecution().getLastExecutionReport().getProcessedItems() -- GitLab From 5de0d5e1042e98481b5307942c0dfbe954d45e72 Mon Sep 17 00:00:00 2001 From: Hugues <hugues.cazeaux@unige.ch> Date: Thu, 6 Feb 2025 17:38:27 +0100 Subject: [PATCH 13/18] feat: upgrade database --- DLCM-Model/src/main/java/ch/dlcm/model/PackageStatus.java | 2 ++ DLCM-Model/src/main/resources/scripts/upgrade22to30.sql | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/DLCM-Model/src/main/java/ch/dlcm/model/PackageStatus.java b/DLCM-Model/src/main/java/ch/dlcm/model/PackageStatus.java index 623bb8723f..e9d50ee2ca 100644 --- a/DLCM-Model/src/main/java/ch/dlcm/model/PackageStatus.java +++ b/DLCM-Model/src/main/java/ch/dlcm/model/PackageStatus.java @@ -32,6 +32,7 @@ import io.swagger.v3.oas.annotations.media.Schema; - CHECK_PENDING => A package verification is pending during checking process - CLEANED => Cleaned package during cleaning process for SIP only - CLEANING => A package clean is in progress during cleaning process for SIP only + - COMPLIANCE_LEVEL_UPDATE_PENDING => A package compliance update is pending - COMPLETED => Completed package - DISPOSABLE => The Package is candidate for disposal process for AIP only - DISPOSAL_APPROVED_BY_ORGUNIT => Disposal org. unit approval done during disposal process for AIP only @@ -58,6 +59,7 @@ import io.swagger.v3.oas.annotations.media.Schema; - RESUBMITTING => A package re-submission is in progress - STORED => Package stored on storage location - TOMBSTONE_REPLICATION_PENDING => A tombstone replication is pending + - UPDATING_COMPLIANCE_LEVEL => A package compliance update is in progress - UPDATING_RETENTION => A package retention update is in progress during disposal process - UPGRADING_METADATA => A metadata version upgrade is in progress """) diff --git a/DLCM-Model/src/main/resources/scripts/upgrade22to30.sql b/DLCM-Model/src/main/resources/scripts/upgrade22to30.sql index 7344689fda..1e85a1af3d 100644 --- a/DLCM-Model/src/main/resources/scripts/upgrade22to30.sql +++ b/DLCM-Model/src/main/resources/scripts/upgrade22to30.sql @@ -33,7 +33,7 @@ ALTER TABLE `additional_fields_form` -- aip ALTER TABLE `aip` - MODIFY COLUMN `status` enum('CHECK_PENDING','CHECKING','CHECKED','CLEANING','CLEANED','COMPLETED','DOWNLOADING','IN_PREPARATION','IN_PROGRESS','STORED','INDEXING','READY','IN_ERROR','PRESERVATION_ERROR','DISPOSABLE','DISPOSAL_APPROVED_BY_ORGUNIT','DISPOSAL_APPROVED','DISPOSED','FIX_PENDING','FIXING','METADATA_EDITION_PENDING','EDITING_METADATA','UPDATING_RETENTION','REINDEXING','RELOADED','RESUBMITTING','PACKAGE_REPLICATION_PENDING','REPLICATING_PACKAGE','TOMBSTONE_REPLICATION_PENDING','REPLICATING_TOMBSTONE', 'METADATA_UPGRADE_PENDING', 'UPGRADING_METADATA') DEFAULT NULL; + MODIFY COLUMN `status` enum('CHECK_PENDING','CHECKING','CHECKED','CLEANING','CLEANED','COMPLETED','DOWNLOADING','IN_PREPARATION','IN_PROGRESS','STORED','INDEXING','READY','IN_ERROR','PRESERVATION_ERROR','DISPOSABLE','DISPOSAL_APPROVED_BY_ORGUNIT','DISPOSAL_APPROVED','DISPOSED','FIX_PENDING','FIXING','METADATA_EDITION_PENDING','EDITING_METADATA','UPDATING_RETENTION','REINDEXING','RELOADED','RESUBMITTING','PACKAGE_REPLICATION_PENDING','REPLICATING_PACKAGE','TOMBSTONE_REPLICATION_PENDING','REPLICATING_TOMBSTONE', 'METADATA_UPGRADE_PENDING', 'UPGRADING_METADATA', 'COMPLIANCE_LEVEL_UPDATE_PENDING', 'UPDATING_COMPLIANCE_LEVEL') DEFAULT NULL; ALTER TABLE `aip` MODIFY COLUMN `data_sensitivity` enum('UNDEFINED','BLUE','GREEN','YELLOW','ORANGE','RED','CRIMSON') DEFAULT NULL; ALTER TABLE `aip` -- GitLab From 2149f14479713898ff1a9288c4abb81a53e5d482 Mon Sep 17 00:00:00 2001 From: Hugues <hugues.cazeaux@unige.ch> Date: Thu, 6 Feb 2025 17:41:01 +0100 Subject: [PATCH 14/18] feat: JMS --- .../main/java/ch/dlcm/business/ArchivalInfoPackageService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/business/ArchivalInfoPackageService.java b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/business/ArchivalInfoPackageService.java index fb4bec4399..e9bf324aeb 100644 --- a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/business/ArchivalInfoPackageService.java +++ b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/business/ArchivalInfoPackageService.java @@ -147,7 +147,7 @@ public class ArchivalInfoPackageService extends InfoPackageService<ArchivalInfoP this.notificationService.createNotification(updaterUser, message, notificationType, organizationalUnit, aip.getResId()); } else if (aip.getInfo().getStatus() == PackageStatus.COMPLETED && (!this.historyService.hasBeenCompleted(aip.getResId()) - || this.historyService.hasBeenUpdatedSinceLastCompletedStatus(aip.getResId()))) { + || this.historyService.hasBeenUpdatedSinceLastCompletedStatus(aip.getResId()))) { message = this.messageService.get(NOTIFICATION_ARCHIVE_COMPLETED, new Object[] { aip.getResId() }); this.notificationService.createArchiveCompletedNotifications(aip, updaterUser, message, organizationalUnit); } @@ -169,6 +169,7 @@ public class ArchivalInfoPackageService extends InfoPackageService<ArchivalInfoP || aip.getInfo().getStatus() == PackageStatus.UPDATING_RETENTION || aip.getInfo().getStatus() == PackageStatus.METADATA_EDITION_PENDING || aip.getInfo().getStatus() == PackageStatus.METADATA_UPGRADE_PENDING + || aip.getInfo().getStatus() == PackageStatus.COMPLIANCE_LEVEL_UPDATE_PENDING || aip.getInfo().getStatus() == PackageStatus.PACKAGE_REPLICATION_PENDING || aip.getInfo().getStatus() == PackageStatus.TOMBSTONE_REPLICATION_PENDING) { boolean isBigPackage = this.getSize(aip) > this.fileSizeLimit.toBytes(); -- GitLab From 31a9f26e8605dfe0bae7f7d96d20e29fa74201da Mon Sep 17 00:00:00 2001 From: Hugues <hugues.cazeaux@unige.ch> Date: Fri, 7 Feb 2025 17:07:17 +0100 Subject: [PATCH 15/18] feat: new status --- .../ch/dlcm/service/AipStatusService.java | 2 +- .../java/ch/dlcm/storage/StorageService.java | 23 +++++++---- .../java/ch/dlcm/model/PackageStatus.java | 3 +- .../java/ch/dlcm/service/MetadataService.java | 16 ++++---- .../service/metadata/MetadataGenerator.java | 7 +++- .../service/metadata/MetadataVersion1.java | 23 ++++++----- .../service/metadata/MetadataVersion2.java | 24 ++++++----- .../service/metadata/MetadataVersion3.java | 27 +++++++------ .../service/metadata/MetadataVersion4.java | 40 ++++++++++--------- 9 files changed, 95 insertions(+), 70 deletions(-) diff --git a/DLCM-ArchivalStorage/src/main/java/ch/dlcm/service/AipStatusService.java b/DLCM-ArchivalStorage/src/main/java/ch/dlcm/service/AipStatusService.java index 7471cf51f5..937eef307f 100644 --- a/DLCM-ArchivalStorage/src/main/java/ch/dlcm/service/AipStatusService.java +++ b/DLCM-ArchivalStorage/src/main/java/ch/dlcm/service/AipStatusService.java @@ -402,7 +402,7 @@ public class AipStatusService extends AbstractAipService { // AIP with UPDATING_COMPLIANCE_LEVEL status private void updateComplianceLevel(ArchivalInfoPackage aip) { try { - if (this.storageService.updateComplianceLevel(aip)) { + if (this.storageService.updateComplianceLevel(aip) > 0) { this.updateAipWithNewPackage(aip); aip.setPackageStatus(PackageStatus.REINDEXING); aip.setLastArchiving(OffsetDateTime.now()); diff --git a/DLCM-ArchivalStorage/src/main/java/ch/dlcm/storage/StorageService.java b/DLCM-ArchivalStorage/src/main/java/ch/dlcm/storage/StorageService.java index 2e8003be05..31a35fad50 100644 --- a/DLCM-ArchivalStorage/src/main/java/ch/dlcm/storage/StorageService.java +++ b/DLCM-ArchivalStorage/src/main/java/ch/dlcm/storage/StorageService.java @@ -55,6 +55,7 @@ import org.springframework.util.MultiValueMap; import ch.unige.solidify.ChecksumAlgorithm; import ch.unige.solidify.SolidifyConstants; +import ch.unige.solidify.config.SolidifyEventPublisher; import ch.unige.solidify.exception.SolidifyCheckingException; import ch.unige.solidify.exception.SolidifyProcessingException; import ch.unige.solidify.service.MessageService; @@ -75,6 +76,7 @@ import ch.dlcm.model.FileInfoUpdate; import ch.dlcm.model.IdentifierType; import ch.dlcm.model.Package; import ch.dlcm.model.PackageStatus; +import ch.dlcm.model.StatusHistory; import ch.dlcm.model.StoredArchiveInterface; import ch.dlcm.model.index.ArchiveMetadata; import ch.dlcm.model.index.ArchivePublicData; @@ -271,7 +273,7 @@ public abstract class StorageService extends DLCMService { } // Update compliance level - public boolean updateComplianceLevel(ArchivalInfoPackage aip) throws IOException, JAXBException, URISyntaxException { + public int updateComplianceLevel(ArchivalInfoPackage aip) throws IOException, JAXBException, URISyntaxException { // Check & extract archive final Path aipPackageFolder = Paths.get(this.checkPackage(aip, false, false)); final Path aipDataFolder = this.getDataFolder(aip.getArchiveContainer(), aipPackageFolder); @@ -286,9 +288,10 @@ public abstract class StorageService extends DLCMService { throw new SolidifyProcessingException("Cannot copy the original metadata file: " + metadataXml + " => " + originalXml); } // Upgrade metadata by adding Premis event - final boolean updated = this.metadataService.updateComplianceLevel(aip, + final int updateNumber = this.metadataService.updateComplianceLevel(aip, aipPackageFolder.resolve(this.getMetadataRelativePath(aip.getArchiveContainer()))); - if (updated) { + if (updateNumber > 0) { + this.updateHistory(aip, PackageStatus.COMPLIANCE_LEVEL_UPDATED.toString(), "Archive compliance level updated (" + updateNumber + ")"); // Generate new package final Path newPackageFile = this.generateNewPackage(aip, aipDataFolder); // Set the archive size @@ -303,7 +306,7 @@ public abstract class StorageService extends DLCMService { if (FileTool.checkFile(aipPackageFolder) && !FileTool.deleteFolder(aipPackageFolder)) { throw new DLCMPreservationException(this.messageService.get(AIP_ERROR_PURGE_FILES_MSG)); } - return updated; + return updateNumber; } // ********************* @@ -793,6 +796,12 @@ public abstract class StorageService extends DLCMService { } } + protected void updateHistory(ArchivalInfoPackage aip, String status, String description) { + final StatusHistory stsHistory = new StatusHistory(aip.getClass().getSimpleName(), aip.getResId(), status); + stsHistory.setDescription(description); + SolidifyEventPublisher.getPublisher().publishEvent(stsHistory); + } + // ********************* // ** Private Methods ** // ********************* @@ -1115,7 +1124,7 @@ public abstract class StorageService extends DLCMService { // First check premis data category anda datatype to take the file object according to it. List<Object> itemList = (List<Object>) ((Map<String, Object>) ((Map<String, Object>) archiveMetadata.getMetadata() .get(DLCMConstants.PREMIS_FIELD)) - .get(DLCMConstants.ITEMS_FIELD)).get(DLCMConstants.ITEM_FIELD); + .get(DLCMConstants.ITEMS_FIELD)).get(DLCMConstants.ITEM_FIELD); for (Object obj : itemList) { Map<String, Object> item = ((Map<String, Object>) obj); if (item.get(DLCMConstants.TYPE_FIELD).equals("dataFile")) { @@ -1183,8 +1192,8 @@ public abstract class StorageService extends DLCMService { // Check all files from thumbnail, readme and dua extension and check if their timestamp from the filename is later from the last time // the aip was completed, meaning the file has being updated since. List<Path> filesToUpdated = allUpdatedFiles.stream().filter(file -> file.getFileName().toString().endsWith(DLCMConstants.THUMBNAIL_EXTENSION) - || file.getFileName().toString().endsWith(DLCMConstants.README_EXTENSION) - || file.getFileName().toString().endsWith(DLCMConstants.DUA_EXTENSION)) + || file.getFileName().toString().endsWith(DLCMConstants.README_EXTENSION) + || file.getFileName().toString().endsWith(DLCMConstants.DUA_EXTENSION)) .filter(file -> { String timestamp = file.getFileName().toString() .substring(file.getFileName().toString().indexOf("-") + 1, file.getFileName().toString().lastIndexOf(".")); diff --git a/DLCM-Model/src/main/java/ch/dlcm/model/PackageStatus.java b/DLCM-Model/src/main/java/ch/dlcm/model/PackageStatus.java index e9d50ee2ca..01b0c9b4c1 100644 --- a/DLCM-Model/src/main/java/ch/dlcm/model/PackageStatus.java +++ b/DLCM-Model/src/main/java/ch/dlcm/model/PackageStatus.java @@ -33,6 +33,7 @@ import io.swagger.v3.oas.annotations.media.Schema; - CLEANED => Cleaned package during cleaning process for SIP only - CLEANING => A package clean is in progress during cleaning process for SIP only - COMPLIANCE_LEVEL_UPDATE_PENDING => A package compliance update is pending + - COMPLIANCE_LEVEL_UPDATED => The compliance levels of the package have been updated - COMPLETED => Completed package - DISPOSABLE => The Package is candidate for disposal process for AIP only - DISPOSAL_APPROVED_BY_ORGUNIT => Disposal org. unit approval done during disposal process for AIP only @@ -73,7 +74,7 @@ public enum PackageStatus { FIX_PENDING, FIXING, METADATA_EDITION_PENDING, EDITING_METADATA, UPDATING_RETENTION, METADATA_UPGRADE_PENDING, UPGRADING_METADATA, - COMPLIANCE_LEVEL_UPDATE_PENDING, UPDATING_COMPLIANCE_LEVEL, + COMPLIANCE_LEVEL_UPDATE_PENDING, UPDATING_COMPLIANCE_LEVEL,COMPLIANCE_LEVEL_UPDATED, REINDEXING, RELOADED, RESUBMITTING, diff --git a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/MetadataService.java b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/MetadataService.java index a248731254..60d4729326 100644 --- a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/MetadataService.java +++ b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/MetadataService.java @@ -33,16 +33,16 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import jakarta.xml.bind.JAXBContext; +import jakarta.xml.bind.JAXBException; +import jakarta.xml.bind.Marshaller; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.logging.LogLevel; import org.springframework.core.io.ClassPathResource; import org.springframework.stereotype.Service; -import jakarta.xml.bind.JAXBContext; -import jakarta.xml.bind.JAXBException; -import jakarta.xml.bind.Marshaller; - import ch.unige.solidify.SolidifyConstants; import ch.unige.solidify.exception.SolidifyCheckingException; import ch.unige.solidify.exception.SolidifyProcessingException; @@ -437,7 +437,7 @@ public class MetadataService extends DLCMService implements OAIMetadataService { metadataGenerator.upgradeMetadata(aip, metadataFile); } - public boolean updateComplianceLevel(ArchivalInfoPackage aip, Path metadataFile) throws JAXBException, IOException { + public int updateComplianceLevel(ArchivalInfoPackage aip, Path metadataFile) throws JAXBException, IOException { // Get metadata generator for the version final MetadataGenerator metadataGenerator = this.getMetadataGenerator(aip.getInfo().getMetadataVersion()); return metadataGenerator.updateCompliance(aip, metadataFile); @@ -469,7 +469,7 @@ public class MetadataService extends DLCMService implements OAIMetadataService { } else { if (df.getStatus() != null && (df.getStatus().equals(DataFileStatus.CHECK_COMPLIANCE) - || df.getStatus().equals(DataFileStatus.CHECK_COMPLIANCE_CLEANED))) { + || df.getStatus().equals(DataFileStatus.CHECK_COMPLIANCE_CLEANED))) { xsd = new ClassPathResource(SolidifyConstants.SCHEMA_HOME + "/" + version.getMetsSchema()); } else { xsd = new ClassPathResource(SolidifyConstants.SCHEMA_HOME + "/" + version.getRepresentationInfoSchema()); @@ -485,8 +485,8 @@ public class MetadataService extends DLCMService implements OAIMetadataService { // Add FITS schema if (!(df instanceof DepositDataFile) || (df.getStatus() != null - && (df.getStatus().equals(DataFileStatus.CHECK_COMPLIANCE) - || df.getStatus().equals(DataFileStatus.CHECK_COMPLIANCE_CLEANED)))) { + && (df.getStatus().equals(DataFileStatus.CHECK_COMPLIANCE) + || df.getStatus().equals(DataFileStatus.CHECK_COMPLIANCE_CLEANED)))) { list.add(new ClassPathResource(SolidifyConstants.SCHEMA_HOME + "/fits_output.xsd").getURL()); } return list; diff --git a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataGenerator.java b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataGenerator.java index a221d3f9c2..49fc69bdc3 100644 --- a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataGenerator.java +++ b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataGenerator.java @@ -262,7 +262,7 @@ public abstract class MetadataGenerator { public abstract void upgradeMetadata(ArchivalInfoPackage aip, Path metadataFile) throws IOException, JAXBException; - public abstract boolean updateCompliance(ArchivalInfoPackage aip, Path metadataFile) throws IOException, JAXBException; + public abstract int updateCompliance(ArchivalInfoPackage aip, Path metadataFile) throws IOException, JAXBException; public abstract Map<String, String> extractInfoFromMetadata(Path metadataFile) throws IOException, JAXBException; @@ -669,6 +669,11 @@ public abstract class MetadataGenerator { if (status.equals(PackageStatus.UPGRADING_METADATA.toString())) { return "updating metadata"; } + // Updating compliance level + if (status.equals(PackageStatus.UPDATING_COMPLIANCE_LEVEL.toString())) { + return "updating compliance level"; + } + return null; } diff --git a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion1.java b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion1.java index 500418e35a..341a11723a 100644 --- a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion1.java +++ b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion1.java @@ -45,20 +45,21 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; + import javax.xml.namespace.QName; import javax.xml.transform.TransformerException; import javax.xml.transform.TransformerFactoryConfigurationError; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.util.StringUtils; - import jakarta.xml.bind.JAXBContext; import jakarta.xml.bind.JAXBElement; import jakarta.xml.bind.JAXBException; import jakarta.xml.bind.Marshaller; import jakarta.xml.bind.Unmarshaller; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.StringUtils; + import ch.unige.solidify.OAIConstants; import ch.unige.solidify.SolidifyConstants; import ch.unige.solidify.exception.SolidifyCheckingException; @@ -1845,7 +1846,7 @@ public class MetadataVersion1 extends MetadataGenerator { case Package -> { switch (df.getDataType()) { case Metadata -> - // DLCM metadata + // DLCM metadata mets.getDmdSec().setMdWrap(this.loadDescriptiveMetadata(df)); case CustomMetadata -> { // Custom Metadata @@ -1858,13 +1859,13 @@ public class MetadataVersion1 extends MetadataGenerator { } } case Primary -> - // Primary data (research data) + // Primary data (research data) this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.RESEARCH_DATA_ROOT, df, outputLocation); case Secondary -> - // Secondary data (documentation) + // Secondary data (documentation) this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.DOC_ROOT, df, outputLocation); case Software -> - // Software + // Software this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.SOFTWARE_ROOT, df, outputLocation); default -> throw new SolidifyProcessingException("Wrong Data Category"); } @@ -1965,8 +1966,10 @@ public class MetadataVersion1 extends MetadataGenerator { } @Override - public boolean updateCompliance(ArchivalInfoPackage aip, Path metadataFile) throws IOException, JAXBException { - return false; + public int updateCompliance(ArchivalInfoPackage aip, Path metadataFile) throws IOException, JAXBException { + // Not applicable for this version + return 0; + } private void updatePreservationPolicy(ArchivalInfoPackage aip, IntellectualEntity premisAip) { diff --git a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion2.java b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion2.java index 082e3e4125..b5c2bdd52c 100644 --- a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion2.java +++ b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion2.java @@ -45,20 +45,21 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; + import javax.xml.namespace.QName; import javax.xml.transform.TransformerException; import javax.xml.transform.TransformerFactoryConfigurationError; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.util.StringUtils; - import jakarta.xml.bind.JAXBContext; import jakarta.xml.bind.JAXBElement; import jakarta.xml.bind.JAXBException; import jakarta.xml.bind.Marshaller; import jakarta.xml.bind.Unmarshaller; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.StringUtils; + import ch.unige.solidify.OAIConstants; import ch.unige.solidify.SolidifyConstants; import ch.unige.solidify.exception.SolidifyCheckingException; @@ -537,8 +538,9 @@ public class MetadataVersion2 extends MetadataGenerator { } @Override - public boolean updateCompliance(ArchivalInfoPackage aip, Path metadataFile) throws IOException, JAXBException { - return false; + public int updateCompliance(ArchivalInfoPackage aip, Path metadataFile) throws IOException, JAXBException { + // Not applicable for this version + return 0; } @Override @@ -1984,7 +1986,7 @@ public class MetadataVersion2 extends MetadataGenerator { case Package -> { switch (df.getDataType()) { case Metadata -> - // DLCM metadata + // DLCM metadata mets.getDmdSec().setMdWrap(this.loadDescriptiveMetadata(df)); case CustomMetadata -> { // Custom Metadata @@ -1997,16 +1999,16 @@ public class MetadataVersion2 extends MetadataGenerator { } } case Primary -> - // Primary data (research data) + // Primary data (research data) this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.RESEARCH_DATA_ROOT, df, outputLocation); case Secondary -> - // Secondary data (documentation) + // Secondary data (documentation) this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.DOC_ROOT, df, outputLocation); case Software -> - // Software + // Software this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.SOFTWARE_ROOT, df, outputLocation); case Internal -> - // Internal (preview files) + // Internal (preview files) this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.INTERNAL_ROOT, df, outputLocation); default -> throw new SolidifyProcessingException("Wrong Data Category"); } diff --git a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion3.java b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion3.java index 19adc55a9b..f83c6c4e98 100644 --- a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion3.java +++ b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion3.java @@ -44,20 +44,21 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; + import javax.xml.namespace.QName; import javax.xml.transform.TransformerException; import javax.xml.transform.TransformerFactoryConfigurationError; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.util.StringUtils; - import jakarta.xml.bind.JAXBContext; import jakarta.xml.bind.JAXBElement; import jakarta.xml.bind.JAXBException; import jakarta.xml.bind.Marshaller; import jakarta.xml.bind.Unmarshaller; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.StringUtils; + import ch.unige.solidify.OAIConstants; import ch.unige.solidify.SolidifyConstants; import ch.unige.solidify.exception.SolidifyCheckingException; @@ -552,8 +553,10 @@ public class MetadataVersion3 extends MetadataGenerator { } @Override - public boolean updateCompliance(ArchivalInfoPackage aip, Path metadataFile) throws IOException, JAXBException { - return false; + public int updateCompliance(ArchivalInfoPackage aip, Path metadataFile) throws IOException, JAXBException { + // Not applicable for this version + return 0; + } @Override @@ -2286,7 +2289,7 @@ public class MetadataVersion3 extends MetadataGenerator { case Package -> { switch (df.getDataType()) { case Metadata -> - // DLCM metadata + // DLCM metadata mets.getDmdSec().setMdWrap(this.loadDescriptiveMetadata(df)); case CustomMetadata -> { // Custom Metadata @@ -2299,19 +2302,19 @@ public class MetadataVersion3 extends MetadataGenerator { } } case Primary -> - // Primary data (research data) + // Primary data (research data) this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.RESEARCH_DATA_ROOT, df, outputLocation); case Secondary -> - // Secondary data (documentation) + // Secondary data (documentation) this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.DOC_ROOT, df, outputLocation); case Software -> - // Software + // Software this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.SOFTWARE_ROOT, df, outputLocation); case Administrative -> - // Administrative data + // Administrative data this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.ADMIN_DATA_ROOT, df, outputLocation); case Internal -> - // Internal (preview files) + // Internal (preview files) this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.INTERNAL_ROOT, df, outputLocation); default -> throw new SolidifyProcessingException("Wrong Data Category"); } diff --git a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion4.java b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion4.java index ed365d4c98..5308f7ebda 100644 --- a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion4.java +++ b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion4.java @@ -50,20 +50,21 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; + import javax.xml.namespace.QName; import javax.xml.transform.TransformerException; import javax.xml.transform.TransformerFactoryConfigurationError; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.util.StringUtils; - import jakarta.xml.bind.JAXBContext; import jakarta.xml.bind.JAXBElement; import jakarta.xml.bind.JAXBException; import jakarta.xml.bind.Marshaller; import jakarta.xml.bind.Unmarshaller; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.StringUtils; + import ch.unige.solidify.OAIConstants; import ch.unige.solidify.SolidifyConstants; import ch.unige.solidify.exception.SolidifyCheckingException; @@ -587,14 +588,15 @@ public class MetadataVersion4 extends MetadataGenerator { } @Override - public boolean updateCompliance(ArchivalInfoPackage aip, Path metadataFile) throws IOException, JAXBException { + public int updateCompliance(ArchivalInfoPackage aip, Path metadataFile) throws IOException, JAXBException { + int count = 0; // Load Mets metadata from file final Mets mets = this.loadMetadataFile(metadataFile); // Get Premis object final PremisComplexType premis = mets.getAmdSec().getDigiprovMD().getMdWrap().getXmlData().getPremis(); // Update compliance level - if (!this.updateComplianceLevel(premis)) { - return false; + if ((count = this.updateComplianceLevel(premis)) == 0) { + return count; } final IntellectualEntity premisAip = this.findObject(premis.getObject(), ResourceName.AIP); // add history event @@ -607,11 +609,11 @@ public class MetadataVersion4 extends MetadataGenerator { } // Save in file this.saveXmlInFile(mets, metadataFile.toString()); - return true; + return count; } - private boolean updateComplianceLevel(PremisComplexType premis) throws JAXBException { - boolean updated = false; + private int updateComplianceLevel(PremisComplexType premis) throws JAXBException { + int count = 0; // Get deposit ID final IntellectualEntity premisDeposit = this.findObject(premis.getObject(), ResourceName.DEPOSIT); // Create Deposit object @@ -642,18 +644,18 @@ public class MetadataVersion4 extends MetadataGenerator { final int newComplianceLevel = this.fileFormatService.assessComplianceLevel(depositDataFile.getResId(), depositDataFile).value(); // Change compliance level if (newComplianceLevel != originalComplianceLevel) { - updated = true; + count++; dlcmInfo.setComplianceLevel(newComplianceLevel); } if (dlcmInfo.getDataClassification().getValue().equals("Unknown")) { - updated = true; + count++; final DepositDataFile df = this.depositRemoteService.getDepositDataFile(deposit.getResId(), depositDataFile.getResId()); dlcmInfo.setDataClassification(this.createDataClassification(df)); } } } } - return updated; + return count; } private PremisComplexType addPremisInfoFromUpdatedFiles(Mets mets, PremisComplexType premis, @@ -2630,7 +2632,7 @@ public class MetadataVersion4 extends MetadataGenerator { case Package -> { switch (df.getDataType()) { case Metadata -> - // DLCM metadata + // DLCM metadata mets.getDmdSec().setMdWrap(this.loadDescriptiveMetadata(df)); case CustomMetadata -> { // Custom Metadata @@ -2643,19 +2645,19 @@ public class MetadataVersion4 extends MetadataGenerator { } } case Primary -> - // Primary data (research data) + // Primary data (research data) this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.RESEARCH_DATA_ROOT, df, outputLocation); case Secondary -> - // Secondary data (documentation) + // Secondary data (documentation) this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.DOC_ROOT, df, outputLocation); case Software -> - // Software + // Software this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.SOFTWARE_ROOT, df, outputLocation); case Administrative -> - // Administrative data + // Administrative data this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.ADMIN_DATA_ROOT, df, outputLocation); case Internal -> - // Internal (preview files) + // Internal (preview files) this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.INTERNAL_ROOT, df, outputLocation); default -> throw new SolidifyProcessingException("Wrong Data Category"); } -- GitLab From 69df693a3247db569098ae244c07e089fd23f753 Mon Sep 17 00:00:00 2001 From: Hugues <hugues.cazeaux@unige.ch> Date: Fri, 7 Feb 2025 22:32:12 +0100 Subject: [PATCH 16/18] feat: new messages --- .../java/ch/dlcm/storage/StorageService.java | 6 ++++-- .../resources/locale/messages_dlcm.properties | 16 +++++++++------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/DLCM-ArchivalStorage/src/main/java/ch/dlcm/storage/StorageService.java b/DLCM-ArchivalStorage/src/main/java/ch/dlcm/storage/StorageService.java index 31a35fad50..4966177534 100644 --- a/DLCM-ArchivalStorage/src/main/java/ch/dlcm/storage/StorageService.java +++ b/DLCM-ArchivalStorage/src/main/java/ch/dlcm/storage/StorageService.java @@ -285,13 +285,15 @@ public abstract class StorageService extends DLCMService { "-" + OffsetDateTime.now().format(DateTimeFormatter.ofPattern(StringTool.DATE_TIME_FORMAT_FOR_FILE)) + SolidifyConstants.XML_EXT)); if (!FileTool.copyFile(metadataXml, originalXml)) { - throw new SolidifyProcessingException("Cannot copy the original metadata file: " + metadataXml + " => " + originalXml); + throw new SolidifyProcessingException( + this.messageService.get("archival.aip.error.metadata-copy", new Object[] { metadataXml, originalXml })); } // Upgrade metadata by adding Premis event final int updateNumber = this.metadataService.updateComplianceLevel(aip, aipPackageFolder.resolve(this.getMetadataRelativePath(aip.getArchiveContainer()))); if (updateNumber > 0) { - this.updateHistory(aip, PackageStatus.COMPLIANCE_LEVEL_UPDATED.toString(), "Archive compliance level updated (" + updateNumber + ")"); + this.updateHistory(aip, PackageStatus.COMPLIANCE_LEVEL_UPDATED.toString(), + this.messageService.get("archival.aip.compliance-updated", new Object[] { updateNumber })); // Generate new package final Path newPackageFile = this.generateNewPackage(aip, aipDataFolder); // Set the archive size diff --git a/DLCM-Common/src/main/resources/locale/messages_dlcm.properties b/DLCM-Common/src/main/resources/locale/messages_dlcm.properties index 46e2bc5633..e6bbe15598 100644 --- a/DLCM-Common/src/main/resources/locale/messages_dlcm.properties +++ b/DLCM-Common/src/main/resources/locale/messages_dlcm.properties @@ -206,8 +206,8 @@ preingest.deposit.status.cancel_editing_metadata=Cancel editing metadata preingest.deposit.status.cleaning=Cleaning preingest.deposit.status.cleaned=Cleaned preingest.deposit.status.deleting=Deposit being deleted -preingest.deposit.status.checking_compliance=Checking compliance level -preingest.deposit.status.error_in_processing=Error when processing internat treatment for the deposit +preingest.deposit.status.checking_compliance=Checking compliance levels +preingest.deposit.status.error_in_processing=Error when processing internal treatment for the deposit preingest.deposit.status.editing_metadata_rejected=Editing metadata was rejected preingest.deposit.status.upgrading_metadata=Upgrading metadata version to the latest preingest.deposit.datafile.notready=Data files of deposit are not ready @@ -237,6 +237,8 @@ ingest.notification.sip.datafile.error=Data files of the SIP {0} are in error archival.aip.completed=The AIP package with ID {0} is already completed archival.aip.forever=The AIP package with ID {0} is archived forever archival.aip.retention=The AIP package with ID {0} is not ready for disposition +archival.aip.compliance-updated=Archive compliance levels updated ({0}) +archival.aip.error.metadata-copy=Cannot copy the original metadata file: {0} => {1} archival.aip.error.collection=A collection cannot be downloaded archival.aip.error.check=During the checking archival.aip.error.process=During the processing @@ -362,7 +364,7 @@ package.status.fixity_error=Fixity error package.status.metadata_upgrade_pending=Metadata version upgrade pending package.status.upgrading_metadata=Upgrading metadata version to the latest package.status.compliance_level_update_pending=Compliance level update pending -package.status.updating_compliance_level=Updating archive compliance level +package.status.updating_compliance_level=Updating archive compliance levels package.error.check_directory=Problem with package directory ({0}) in {1} ############# # Data file # @@ -393,10 +395,10 @@ datafile.status.processed=Downloaded datafile.status.ready=Ready datafile.status.cleaning=Cleaning datafile.status.cleaned=Cleaned -datafile.status.check_compliance=Check compliance level -datafile.status.check_compliance_clean=Check compliance level for cleaned data file -datafile.status.checked_compliance=Checked compliance level -datafile.status.checked_compliance_clean=Checked compliance level for cleaned data file +datafile.status.check_compliance=Check compliance levels +datafile.status.check_compliance_clean=Check compliance levels for cleaned data file +datafile.status.checked_compliance=Checked compliance levels +datafile.status.checked_compliance_clean=Checked compliance levels for cleaned data file datafile.status.change_relative_location=Change relative location datafile.status.change_data_category=Change data category datafile.status.file_format_unknown=No found format by {0} -- GitLab From de235b38f602accf2276474a6720f878bad78a14 Mon Sep 17 00:00:00 2001 From: Hugues <hugues.cazeaux@unige.ch> Date: Mon, 10 Feb 2025 11:50:52 +0100 Subject: [PATCH 17/18] chore: fixed format --- .../resources/xslt/dlcm4migration-2.1-3.0.xsl | 2 +- .../resources/xslt/dlcm4migration-3.0-3.1.xsl | 2 +- .../resources/xslt/dlcm4migration-3.1-4.0.xsl | 2 +- .../service/metadata/MetadataVersion1.java | 28 +++++++------ .../service/metadata/MetadataVersion2.java | 35 +++++++++------- .../service/metadata/MetadataVersion3.java | 42 +++++++++++-------- .../service/metadata/MetadataVersion4.java | 42 +++++++++++-------- 7 files changed, 87 insertions(+), 66 deletions(-) diff --git a/DLCM-Model/src/main/resources/xslt/dlcm4migration-2.1-3.0.xsl b/DLCM-Model/src/main/resources/xslt/dlcm4migration-2.1-3.0.xsl index e5f003ab33..9abdff5703 100644 --- a/DLCM-Model/src/main/resources/xslt/dlcm4migration-2.1-3.0.xsl +++ b/DLCM-Model/src/main/resources/xslt/dlcm4migration-2.1-3.0.xsl @@ -1,7 +1,7 @@ <?xml version="1.0" encoding="UTF-8"?> <xsl:stylesheet version="2.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:mets="http://www.loc.gov/METS/" xmlns:datacite="http://datacite.org/schema/kernel-4" xmlns:premis="http://www.loc.gov/premis/v3" - xmlns:dlcm="http://www.dlcm.ch/dlcm/v2" xmlns:fits="http://hul.harvard.edu/ois/xml/ns/fits/fits_output" exclude-result-prefixes=""> + xmlns:dlcm="http://www.dlcm.ch/dlcm/v2" xmlns:fits="http://hul.harvard.edu/ois/xml/ns/fits/fits_output"> <xsl:output method="xml" omit-xml-declaration="no" indent="yes" encoding="UTF-8" /> diff --git a/DLCM-Model/src/main/resources/xslt/dlcm4migration-3.0-3.1.xsl b/DLCM-Model/src/main/resources/xslt/dlcm4migration-3.0-3.1.xsl index 3d308a727e..72fa63902d 100644 --- a/DLCM-Model/src/main/resources/xslt/dlcm4migration-3.0-3.1.xsl +++ b/DLCM-Model/src/main/resources/xslt/dlcm4migration-3.0-3.1.xsl @@ -1,7 +1,7 @@ <?xml version="1.0" encoding="UTF-8"?> <xsl:stylesheet version="2.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:mets="http://www.loc.gov/METS/" xmlns:datacite="http://datacite.org/schema/kernel-4" xmlns:premis="http://www.loc.gov/premis/v3" - xmlns:dlcm="http://www.dlcm.ch/dlcm/v2" xmlns:fits="http://hul.harvard.edu/ois/xml/ns/fits/fits_output" exclude-result-prefixes=""> + xmlns:dlcm="http://www.dlcm.ch/dlcm/v2" xmlns:fits="http://hul.harvard.edu/ois/xml/ns/fits/fits_output"> <xsl:output method="xml" omit-xml-declaration="no" indent="yes" encoding="UTF-8" /> diff --git a/DLCM-Model/src/main/resources/xslt/dlcm4migration-3.1-4.0.xsl b/DLCM-Model/src/main/resources/xslt/dlcm4migration-3.1-4.0.xsl index 09a2a79568..43156f814c 100644 --- a/DLCM-Model/src/main/resources/xslt/dlcm4migration-3.1-4.0.xsl +++ b/DLCM-Model/src/main/resources/xslt/dlcm4migration-3.1-4.0.xsl @@ -2,7 +2,7 @@ <xsl:stylesheet version="2.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:mets="http://www.loc.gov/METS/" xmlns:datacite="http://datacite.org/schema/kernel-4" xmlns:premis="http://www.loc.gov/premis/v3" xmlns:dlcm2="http://www.dlcm.ch/dlcm/v2" xmlns:dlcm="http://www.dlcm.ch/dlcm/v3" xmlns:fits="http://hul.harvard.edu/ois/xml/ns/fits/fits_output" - exclude-result-prefixes=""> + exclude-result-prefixes="dlcm2"> <xsl:output method="xml" omit-xml-declaration="no" indent="yes" encoding="UTF-8" /> diff --git a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion1.java b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion1.java index 341a11723a..df0c67b847 100644 --- a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion1.java +++ b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion1.java @@ -1845,9 +1845,10 @@ public class MetadataVersion1 extends MetadataGenerator { switch (df.getDataCategory()) { case Package -> { switch (df.getDataType()) { - case Metadata -> - // DLCM metadata - mets.getDmdSec().setMdWrap(this.loadDescriptiveMetadata(df)); + case Metadata -> { + // DLCM metadata + mets.getDmdSec().setMdWrap(this.loadDescriptiveMetadata(df)); + } case CustomMetadata -> { // Custom Metadata this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.METADATA_ROOT, df, outputLocation); @@ -1858,15 +1859,18 @@ public class MetadataVersion1 extends MetadataGenerator { default -> throw new SolidifyProcessingException("Wrong Data Type"); } } - case Primary -> - // Primary data (research data) - this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.RESEARCH_DATA_ROOT, df, outputLocation); - case Secondary -> - // Secondary data (documentation) - this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.DOC_ROOT, df, outputLocation); - case Software -> - // Software - this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.SOFTWARE_ROOT, df, outputLocation); + case Primary -> { + // Primary data (research data) + this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.RESEARCH_DATA_ROOT, df, outputLocation); + } + case Secondary -> { + // Secondary data (documentation) + this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.DOC_ROOT, df, outputLocation); + } + case Software -> { + // Software + this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.SOFTWARE_ROOT, df, outputLocation); + } default -> throw new SolidifyProcessingException("Wrong Data Category"); } nbProcessedDataFiles++; diff --git a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion2.java b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion2.java index b5c2bdd52c..3bc61b4499 100644 --- a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion2.java +++ b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion2.java @@ -1985,9 +1985,10 @@ public class MetadataVersion2 extends MetadataGenerator { switch (df.getDataCategory()) { case Package -> { switch (df.getDataType()) { - case Metadata -> - // DLCM metadata - mets.getDmdSec().setMdWrap(this.loadDescriptiveMetadata(df)); + case Metadata -> { + // DLCM metadata + mets.getDmdSec().setMdWrap(this.loadDescriptiveMetadata(df)); + } case CustomMetadata -> { // Custom Metadata this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.METADATA_ROOT, df, outputLocation); @@ -1998,18 +1999,22 @@ public class MetadataVersion2 extends MetadataGenerator { default -> throw new SolidifyProcessingException("Wrong Data Type"); } } - case Primary -> - // Primary data (research data) - this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.RESEARCH_DATA_ROOT, df, outputLocation); - case Secondary -> - // Secondary data (documentation) - this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.DOC_ROOT, df, outputLocation); - case Software -> - // Software - this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.SOFTWARE_ROOT, df, outputLocation); - case Internal -> - // Internal (preview files) - this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.INTERNAL_ROOT, df, outputLocation); + case Primary -> { + // Primary data (research data) + this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.RESEARCH_DATA_ROOT, df, outputLocation); + } + case Secondary -> { + // Secondary data (documentation) + this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.DOC_ROOT, df, outputLocation); + } + case Software -> { + // Software + this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.SOFTWARE_ROOT, df, outputLocation); + } + case Internal -> { + // Internal (metadata files) + this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.INTERNAL_ROOT, df, outputLocation); + } default -> throw new SolidifyProcessingException("Wrong Data Category"); } nbProcessedDataFiles++; diff --git a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion3.java b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion3.java index f83c6c4e98..92236ec32b 100644 --- a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion3.java +++ b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion3.java @@ -2288,9 +2288,10 @@ public class MetadataVersion3 extends MetadataGenerator { switch (df.getDataCategory()) { case Package -> { switch (df.getDataType()) { - case Metadata -> - // DLCM metadata - mets.getDmdSec().setMdWrap(this.loadDescriptiveMetadata(df)); + case Metadata -> { + // DLCM metadata + mets.getDmdSec().setMdWrap(this.loadDescriptiveMetadata(df)); + } case CustomMetadata -> { // Custom Metadata this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.METADATA_ROOT, df, outputLocation); @@ -2301,21 +2302,26 @@ public class MetadataVersion3 extends MetadataGenerator { default -> throw new SolidifyProcessingException("Wrong Data Type"); } } - case Primary -> - // Primary data (research data) - this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.RESEARCH_DATA_ROOT, df, outputLocation); - case Secondary -> - // Secondary data (documentation) - this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.DOC_ROOT, df, outputLocation); - case Software -> - // Software - this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.SOFTWARE_ROOT, df, outputLocation); - case Administrative -> - // Administrative data - this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.ADMIN_DATA_ROOT, df, outputLocation); - case Internal -> - // Internal (preview files) - this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.INTERNAL_ROOT, df, outputLocation); + case Primary -> { + // Primary data (research data) + this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.RESEARCH_DATA_ROOT, df, outputLocation); + } + case Secondary -> { + // Secondary data (documentation) + this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.DOC_ROOT, df, outputLocation); + } + case Software -> { + // Software + this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.SOFTWARE_ROOT, df, outputLocation); + } + case Administrative -> { + // Administrative data + this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.ADMIN_DATA_ROOT, df, outputLocation); + } + case Internal -> { + // Internal (metadata files) + this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.INTERNAL_ROOT, df, outputLocation); + } default -> throw new SolidifyProcessingException("Wrong Data Category"); } nbProcessedDataFiles++; diff --git a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion4.java b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion4.java index 5308f7ebda..2e6f4c78b9 100644 --- a/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion4.java +++ b/DLCM-ResourceServerCommon/src/main/java/ch/dlcm/service/metadata/MetadataVersion4.java @@ -2631,9 +2631,10 @@ public class MetadataVersion4 extends MetadataGenerator { switch (df.getDataCategory()) { case Package -> { switch (df.getDataType()) { - case Metadata -> - // DLCM metadata - mets.getDmdSec().setMdWrap(this.loadDescriptiveMetadata(df)); + case Metadata -> { + // DLCM metadata + mets.getDmdSec().setMdWrap(this.loadDescriptiveMetadata(df)); + } case CustomMetadata -> { // Custom Metadata this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.METADATA_ROOT, df, outputLocation); @@ -2644,21 +2645,26 @@ public class MetadataVersion4 extends MetadataGenerator { default -> throw new SolidifyProcessingException("Wrong Data Type"); } } - case Primary -> - // Primary data (research data) - this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.RESEARCH_DATA_ROOT, df, outputLocation); - case Secondary -> - // Secondary data (documentation) - this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.DOC_ROOT, df, outputLocation); - case Software -> - // Software - this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.SOFTWARE_ROOT, df, outputLocation); - case Administrative -> - // Administrative data - this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.ADMIN_DATA_ROOT, df, outputLocation); - case Internal -> - // Internal (preview files) - this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.INTERNAL_ROOT, df, outputLocation); + case Primary -> { + // Primary data (research data) + this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.RESEARCH_DATA_ROOT, df, outputLocation); + } + case Secondary -> { + // Secondary data (documentation) + this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.DOC_ROOT, df, outputLocation); + } + case Software -> { + // Software + this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.SOFTWARE_ROOT, df, outputLocation); + } + case Administrative -> { + // Administrative data + this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.ADMIN_DATA_ROOT, df, outputLocation); + } + case Internal -> { + // Internal (metadata files) + this.loadFile(root, structMap.getDiv(), premis, DLCMConstants.INTERNAL_ROOT, df, outputLocation); + } default -> throw new SolidifyProcessingException("Wrong Data Category"); } nbProcessedDataFiles++; -- GitLab From 0ccaf591b513edac7e09cdcc9b44bd090c5d3004 Mon Sep 17 00:00:00 2001 From: Hugues <hugues.cazeaux@unige.ch> Date: Mon, 10 Feb 2025 11:54:37 +0100 Subject: [PATCH 18/18] chore: fixed format --- .../src/main/java/ch/dlcm/storage/StorageService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DLCM-ArchivalStorage/src/main/java/ch/dlcm/storage/StorageService.java b/DLCM-ArchivalStorage/src/main/java/ch/dlcm/storage/StorageService.java index 4966177534..2a21b24a19 100644 --- a/DLCM-ArchivalStorage/src/main/java/ch/dlcm/storage/StorageService.java +++ b/DLCM-ArchivalStorage/src/main/java/ch/dlcm/storage/StorageService.java @@ -1126,7 +1126,8 @@ public abstract class StorageService extends DLCMService { // First check premis data category anda datatype to take the file object according to it. List<Object> itemList = (List<Object>) ((Map<String, Object>) ((Map<String, Object>) archiveMetadata.getMetadata() .get(DLCMConstants.PREMIS_FIELD)) - .get(DLCMConstants.ITEMS_FIELD)).get(DLCMConstants.ITEM_FIELD); + .get(DLCMConstants.ITEMS_FIELD)) + .get(DLCMConstants.ITEM_FIELD); for (Object obj : itemList) { Map<String, Object> item = ((Map<String, Object>) obj); if (item.get(DLCMConstants.TYPE_FIELD).equals("dataFile")) { -- GitLab