From 82d641cb88f57ca72fe26047721e9b391027cc53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20ROUCOU?= Date: Wed, 10 Jan 2024 17:35:30 +0100 Subject: [PATCH 001/137] Migrate HttpSolrClient to Http2SolrClient with refactoring in order to use one service to query solr, and another service to manage index --- .../edu/harvard/iq/dataverse/DatasetPage.java | 4 +- .../search/AbstractSolrClientService.java | 46 +++++++++ .../iq/dataverse/search/IndexServiceBean.java | 93 ++++++++----------- .../search/SolrClientIndexService.java | 46 +++++++++ .../dataverse/search/SolrClientService.java | 36 +------ .../search/SolrIndexServiceBean.java | 4 +- .../search/IndexServiceBeanTest.java | 59 +++++------- .../search/SolrClientIndexServiceTest.java | 42 +++++++++ .../search/SolrClientServiceTest.java | 6 +- 9 files changed, 208 insertions(+), 128 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/search/AbstractSolrClientService.java create mode 100644 src/main/java/edu/harvard/iq/dataverse/search/SolrClientIndexService.java create mode 100644 src/test/java/edu/harvard/iq/dataverse/search/SolrClientIndexServiceTest.java diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index b79f387f20b..b720b6a1b5c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -154,7 +154,7 @@ import edu.harvard.iq.dataverse.util.FileMetadataUtil; import java.util.Comparator; import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.impl.BaseHttpSolrClient.RemoteSolrException; import org.apache.solr.client.solrj.response.FacetField; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrDocument; @@ -991,7 +991,7 @@ public Set getFileIdsInVersionFromSolr(Long datasetVersionId, String patte try { queryResponse = solrClientService.getSolrClient().query(solrQuery); - } catch (HttpSolrClient.RemoteSolrException ex) { + } catch (RemoteSolrException ex) { logger.fine("Remote Solr Exception: " + ex.getLocalizedMessage()); String msg = ex.getLocalizedMessage(); if (msg.contains(SearchFields.FILE_DELETED)) { diff --git a/src/main/java/edu/harvard/iq/dataverse/search/AbstractSolrClientService.java b/src/main/java/edu/harvard/iq/dataverse/search/AbstractSolrClientService.java new file mode 100644 index 00000000000..f36c4a9e591 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/search/AbstractSolrClientService.java @@ -0,0 +1,46 @@ +package edu.harvard.iq.dataverse.search; + +import java.io.IOException; +import java.util.logging.Logger; + +import org.apache.solr.client.solrj.SolrClient; + +import edu.harvard.iq.dataverse.settings.JvmSettings; +import edu.harvard.iq.dataverse.util.SystemConfig; +import jakarta.ejb.EJB; + +public abstract class AbstractSolrClientService { + private static final Logger logger = Logger.getLogger(AbstractSolrClientService.class.getCanonicalName()); + + @EJB + SystemConfig systemConfig; + + public abstract void init(); + public abstract void close(); + public abstract SolrClient getSolrClient(); + public abstract void setSolrClient(SolrClient solrClient); + + public void close(SolrClient solrClient) { + if (solrClient != null) { + try { + solrClient.close(); + } catch (IOException e) { + logger.warning("Solr closing error: " + e); + } + solrClient = null; + } + } + + public void reInitialize() { + close(); + init(); + } + + public String getSolrUrl() { + // Get from MPCONFIG. Might be configured by a sysadmin or simply return the + // default shipped with resources/META-INF/microprofile-config.properties. + final String protocol = JvmSettings.SOLR_PROT.lookup(); + final String path = JvmSettings.SOLR_PATH.lookup(); + return protocol + "://" + this.systemConfig.getSolrHostColonPort() + path; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index 9e73c38a5d0..189cfb5de6c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -1,6 +1,28 @@ package edu.harvard.iq.dataverse.search; -import edu.harvard.iq.dataverse.*; +import edu.harvard.iq.dataverse.ControlledVocabularyValue; +import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.DataFileServiceBean; +import edu.harvard.iq.dataverse.DataFileTag; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetField; +import edu.harvard.iq.dataverse.DatasetFieldCompoundValue; +import edu.harvard.iq.dataverse.DatasetFieldConstant; +import edu.harvard.iq.dataverse.DatasetFieldServiceBean; +import edu.harvard.iq.dataverse.DatasetFieldType; +import edu.harvard.iq.dataverse.DatasetFieldValueValidator; +import edu.harvard.iq.dataverse.DatasetLinkingServiceBean; +import edu.harvard.iq.dataverse.DatasetServiceBean; +import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.DataverseLinkingServiceBean; +import edu.harvard.iq.dataverse.DataverseServiceBean; +import edu.harvard.iq.dataverse.DvObject; +import edu.harvard.iq.dataverse.DvObjectServiceBean; +import edu.harvard.iq.dataverse.Embargo; +import edu.harvard.iq.dataverse.FileMetadata; +import edu.harvard.iq.dataverse.GlobalId; +import edu.harvard.iq.dataverse.PermissionServiceBean; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUserServiceBean; import edu.harvard.iq.dataverse.batch.util.LoggingUtil; @@ -14,7 +36,6 @@ import edu.harvard.iq.dataverse.harvest.client.HarvestingClient; import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; -import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.StringUtil; import edu.harvard.iq.dataverse.util.SystemConfig; @@ -39,8 +60,6 @@ import java.util.function.Function; import java.util.logging.Logger; import java.util.stream.Collectors; -import jakarta.annotation.PostConstruct; -import jakarta.annotation.PreDestroy; import jakarta.ejb.AsyncResult; import jakarta.ejb.Asynchronous; import jakarta.ejb.EJB; @@ -55,11 +74,9 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrQuery.SortClause; import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.response.UpdateResponse; import org.apache.solr.common.SolrDocument; @@ -109,16 +126,15 @@ public class IndexServiceBean { @EJB SettingsServiceBean settingsService; @EJB - SolrClientService solrClientService; + SolrClientService solrClientService; // only for query index on Solr + @EJB + SolrClientIndexService solrClientIndexService; // only for add, update, or remove index on Solr @EJB DataFileServiceBean dataFileService; @EJB VariableServiceBean variableService; - - @EJB - IndexBatchServiceBean indexBatchService; - + @EJB DatasetFieldServiceBean datasetFieldService; @@ -138,37 +154,10 @@ public class IndexServiceBean { private static final String IN_REVIEW_STRING = "In Review"; private static final String DEACCESSIONED_STRING = "Deaccessioned"; public static final String HARVESTED = "Harvested"; - private String rootDataverseName; private Dataverse rootDataverseCached; - SolrClient solrServer; private VariableMetadataUtil variableMetadataUtil; - @PostConstruct - public void init() { - // Get from MPCONFIG. Might be configured by a sysadmin or simply return the default shipped with - // resources/META-INF/microprofile-config.properties. - String protocol = JvmSettings.SOLR_PROT.lookup(); - String path = JvmSettings.SOLR_PATH.lookup(); - - String urlString = protocol + "://" + systemConfig.getSolrHostColonPort() + path; - solrServer = new HttpSolrClient.Builder(urlString).build(); - - rootDataverseName = findRootDataverseCached().getName(); - } - - @PreDestroy - public void close() { - if (solrServer != null) { - try { - solrServer.close(); - } catch (IOException e) { - logger.warning("Solr closing error: " + e); - } - solrServer = null; - } - } - @TransactionAttribute(REQUIRES_NEW) public Future indexDataverseInNewTransaction(Dataverse dataverse) throws SolrServerException, IOException{ return indexDataverse(dataverse, false); @@ -303,7 +292,7 @@ public Future indexDataverse(Dataverse dataverse, boolean processPaths) String status; try { if (dataverse.getId() != null) { - solrClientService.getSolrClient().add(docs); + solrClientIndexService.getSolrClient().add(docs); } else { logger.info("WARNING: indexing of a dataverse with no id attempted"); } @@ -313,7 +302,7 @@ public Future indexDataverse(Dataverse dataverse, boolean processPaths) return new AsyncResult<>(status); } try { - solrClientService.getSolrClient().commit(); + solrClientIndexService.getSolrClient().commit(); } catch (SolrServerException | IOException ex) { status = ex.toString(); logger.info(status); @@ -1447,8 +1436,8 @@ private String addOrUpdateDataset(IndexableDataset indexableDataset, Set d final SolrInputDocuments docs = toSolrDocs(indexableDataset, datafilesInDraftVersion); try { - solrClientService.getSolrClient().add(docs.getDocuments()); - solrClientService.getSolrClient().commit(); + solrClientIndexService.getSolrClient().add(docs.getDocuments()); + solrClientIndexService.getSolrClient().commit(); } catch (SolrServerException | IOException ex) { if (ex.getCause() instanceof SolrServerException) { throw new SolrServerException(ex); @@ -1689,8 +1678,8 @@ private void updatePathForExistingSolrDocs(DvObject object) throws SolrServerExc sid.removeField(SearchFields.SUBTREE); sid.addField(SearchFields.SUBTREE, paths); - UpdateResponse addResponse = solrClientService.getSolrClient().add(sid); - UpdateResponse commitResponse = solrClientService.getSolrClient().commit(); + UpdateResponse addResponse = solrClientIndexService.getSolrClient().add(sid); + UpdateResponse commitResponse = solrClientIndexService.getSolrClient().commit(); if (object.isInstanceofDataset()) { for (DataFile df : dataset.getFiles()) { solrQuery.setQuery(SearchUtil.constructQuery(SearchFields.ENTITY_ID, df.getId().toString())); @@ -1703,8 +1692,8 @@ private void updatePathForExistingSolrDocs(DvObject object) throws SolrServerExc } sid.removeField(SearchFields.SUBTREE); sid.addField(SearchFields.SUBTREE, paths); - addResponse = solrClientService.getSolrClient().add(sid); - commitResponse = solrClientService.getSolrClient().commit(); + addResponse = solrClientIndexService.getSolrClient().add(sid); + commitResponse = solrClientIndexService.getSolrClient().commit(); } } } @@ -1746,12 +1735,12 @@ public String delete(Dataverse doomed) { logger.fine("deleting Solr document for dataverse " + doomed.getId()); UpdateResponse updateResponse; try { - updateResponse = solrClientService.getSolrClient().deleteById(solrDocIdentifierDataverse + doomed.getId()); + updateResponse = solrClientIndexService.getSolrClient().deleteById(solrDocIdentifierDataverse + doomed.getId()); } catch (SolrServerException | IOException ex) { return ex.toString(); } try { - solrClientService.getSolrClient().commit(); + solrClientIndexService.getSolrClient().commit(); } catch (SolrServerException | IOException ex) { return ex.toString(); } @@ -1771,12 +1760,12 @@ public String removeSolrDocFromIndex(String doomed) { logger.fine("deleting Solr document: " + doomed); UpdateResponse updateResponse; try { - updateResponse = solrClientService.getSolrClient().deleteById(doomed); + updateResponse = solrClientIndexService.getSolrClient().deleteById(doomed); } catch (SolrServerException | IOException ex) { return ex.toString(); } try { - solrClientService.getSolrClient().commit(); + solrClientIndexService.getSolrClient().commit(); } catch (SolrServerException | IOException ex) { return ex.toString(); } @@ -1968,7 +1957,7 @@ public List findPermissionsInSolrOnly() throws SearchException { boolean done = false; while (!done) { q.set(CursorMarkParams.CURSOR_MARK_PARAM, cursorMark); - QueryResponse rsp = solrServer.query(q); + QueryResponse rsp = solrClientService.getSolrClient().query(q); String nextCursorMark = rsp.getNextCursorMark(); SolrDocumentList list = rsp.getResults(); for (SolrDocument doc: list) { @@ -2003,7 +1992,7 @@ private List findDvObjectInSolrOnly(String type) throws SearchException solrQuery.set(CursorMarkParams.CURSOR_MARK_PARAM, cursorMark); QueryResponse rsp = null; try { - rsp = solrServer.query(solrQuery); + rsp = solrClientService.getSolrClient().query(solrQuery); } catch (SolrServerException | IOException ex) { throw new SearchException("Error searching Solr type: " + type, ex); diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrClientIndexService.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrClientIndexService.java new file mode 100644 index 00000000000..6a98c7cd4a0 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrClientIndexService.java @@ -0,0 +1,46 @@ +package edu.harvard.iq.dataverse.search; + +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient; +import org.apache.solr.client.solrj.impl.Http2SolrClient; + +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import jakarta.ejb.Singleton; +import jakarta.inject.Named; +import java.util.logging.Logger; + +/** + * Solr client to provide insert/update/delete operations. + * Don't use this service with queries to Solr, use SolrClientService instend. + */ +@Named +@Singleton +public class SolrClientIndexService extends AbstractSolrClientService { + private static final Logger logger = Logger.getLogger(SolrClientIndexService.class.getCanonicalName()); + + private SolrClient solrClient; + + @PostConstruct + public void init() { + solrClient = new ConcurrentUpdateHttp2SolrClient.Builder( + getSolrUrl(), new Http2SolrClient.Builder().build()).build(); + } + + @PreDestroy + public void close() { + close(solrClient); + } + + public SolrClient getSolrClient() { + // Should never happen - but? + if (solrClient == null) { + init(); + } + return solrClient; + } + + public void setSolrClient(SolrClient solrClient) { + this.solrClient = solrClient; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrClientService.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrClientService.java index b36130de7c8..60ab269455a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrClientService.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrClientService.java @@ -5,24 +5,20 @@ */ package edu.harvard.iq.dataverse.search; -import edu.harvard.iq.dataverse.settings.JvmSettings; -import edu.harvard.iq.dataverse.util.SystemConfig; import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.impl.Http2SolrClient; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; -import jakarta.ejb.EJB; import jakarta.ejb.Singleton; import jakarta.inject.Named; -import java.io.IOException; import java.util.logging.Logger; /** * * @author landreev * - * This singleton is dedicated to initializing the HttpSolrClient used by the + * This singleton is dedicated to initializing the Http2SolrClient used by the * application to talk to the search engine, and serving it to all the other * classes that need it. * This ensures that we are using one client only - as recommended by the @@ -30,36 +26,19 @@ */ @Named @Singleton -public class SolrClientService { +public class SolrClientService extends AbstractSolrClientService { private static final Logger logger = Logger.getLogger(SolrClientService.class.getCanonicalName()); - @EJB - SystemConfig systemConfig; - private SolrClient solrClient; @PostConstruct public void init() { - // Get from MPCONFIG. Might be configured by a sysadmin or simply return the default shipped with - // resources/META-INF/microprofile-config.properties. - String protocol = JvmSettings.SOLR_PROT.lookup(); - String path = JvmSettings.SOLR_PATH.lookup(); - - String urlString = protocol + "://" + systemConfig.getSolrHostColonPort() + path; - solrClient = new HttpSolrClient.Builder(urlString).build(); + solrClient = new Http2SolrClient.Builder(getSolrUrl()).build(); } @PreDestroy public void close() { - if (solrClient != null) { - try { - solrClient.close(); - } catch (IOException e) { - logger.warning("Solr closing error: " + e); - } - - solrClient = null; - } + close(solrClient); } public SolrClient getSolrClient() { @@ -73,9 +52,4 @@ public SolrClient getSolrClient() { public void setSolrClient(SolrClient solrClient) { this.solrClient = solrClient; } - - public void reInitialize() { - close(); - init(); - } } diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java index 04021eb75b6..dfc1ced7c12 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java @@ -46,9 +46,7 @@ public class SolrIndexServiceBean { @EJB DataverseRoleServiceBean rolesSvc; @EJB - IndexServiceBean indexService; - @EJB - SolrClientService solrClientService; + SolrClientIndexService solrClientService; public static String numRowsClearedByClearAllIndexTimes = "numRowsClearedByClearAllIndexTimes"; public static String messageString = "message"; diff --git a/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java index adf48e05f09..a9ef468d163 100644 --- a/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java @@ -1,16 +1,26 @@ package edu.harvard.iq.dataverse.search; -import edu.harvard.iq.dataverse.*; +import edu.harvard.iq.dataverse.ControlledVocabularyValue; +import edu.harvard.iq.dataverse.DOIServiceBean; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetField; +import edu.harvard.iq.dataverse.DatasetFieldCompoundValue; +import edu.harvard.iq.dataverse.DatasetFieldConstant; +import edu.harvard.iq.dataverse.DatasetFieldServiceBean; +import edu.harvard.iq.dataverse.DatasetFieldType; +import edu.harvard.iq.dataverse.DatasetFieldValue; +import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.Dataverse.DataverseType; +import edu.harvard.iq.dataverse.DataverseServiceBean; +import edu.harvard.iq.dataverse.GlobalId; +import edu.harvard.iq.dataverse.MetadataBlock; import edu.harvard.iq.dataverse.branding.BrandingUtil; import edu.harvard.iq.dataverse.mocks.MocksFactory; -import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.SystemConfig; -import edu.harvard.iq.dataverse.util.testing.JvmSetting; import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings; import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.common.SolrInputDocument; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -21,11 +31,15 @@ import org.mockito.junit.jupiter.MockitoExtension; import java.io.IOException; -import java.util.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Optional; +import java.util.Set; import java.util.logging.Logger; import java.util.stream.Collectors; -import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @LocalJvmSettings @@ -40,7 +54,7 @@ public class IndexServiceBeanTest { private SettingsServiceBean settingsService; @InjectMocks private SystemConfig systemConfig = new SystemConfig(); - + @BeforeEach public void setUp() { dataverse = MocksFactory.makeDataverse(); @@ -54,36 +68,6 @@ public void setUp() { Mockito.when(indexService.dataverseService.findRootDataverse()).thenReturn(dataverse); } - - @Test - public void testInitWithDefaults() { - // given - String url = "http://localhost:8983/solr/collection1"; - - // when - indexService.init(); - - // then - HttpSolrClient client = (HttpSolrClient) indexService.solrServer; - assertEquals(url, client.getBaseURL()); - } - - - @Test - @JvmSetting(key = JvmSettings.SOLR_HOST, value = "foobar") - @JvmSetting(key = JvmSettings.SOLR_PORT, value = "1234") - @JvmSetting(key = JvmSettings.SOLR_CORE, value = "test") - void testInitWithConfig() { - // given - String url = "http://foobar:1234/solr/test"; - - // when - indexService.init(); - - // then - HttpSolrClient client = (HttpSolrClient) indexService.solrServer; - assertEquals(url, client.getBaseURL()); - } @Test public void TestIndexing() throws SolrServerException, IOException { @@ -125,6 +109,7 @@ public void testValidateBoundingBox() throws SolrServerException, IOException { assertTrue(!doc.get().containsKey("geolocation")); assertTrue(!doc.get().containsKey("boundingBox")); } + private DatasetField constructBoundingBoxValue(String datasetFieldTypeName, String value) { DatasetField retVal = new DatasetField(); retVal.setDatasetFieldType(new DatasetFieldType(datasetFieldTypeName, DatasetFieldType.FieldType.TEXT, false)); diff --git a/src/test/java/edu/harvard/iq/dataverse/search/SolrClientIndexServiceTest.java b/src/test/java/edu/harvard/iq/dataverse/search/SolrClientIndexServiceTest.java new file mode 100644 index 00000000000..d6459626faf --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/search/SolrClientIndexServiceTest.java @@ -0,0 +1,42 @@ +package edu.harvard.iq.dataverse.search; + +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import edu.harvard.iq.dataverse.util.SystemConfig; +import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings; + +import org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +@LocalJvmSettings +@ExtendWith(MockitoExtension.class) +class SolrClientIndexServiceTest { + + @Mock + SettingsServiceBean settingsServiceBean; + @InjectMocks + SystemConfig systemConfig; + SolrClientIndexService clientService = new SolrClientIndexService(); + + @BeforeEach + void setUp() { + clientService.systemConfig = systemConfig; + } + + @Test + void testInitWithDefaults() { + // when + clientService.init(); + + // then + ConcurrentUpdateHttp2SolrClient client = (ConcurrentUpdateHttp2SolrClient) clientService.getSolrClient(); + assertNotNull(client); + } + +} \ No newline at end of file diff --git a/src/test/java/edu/harvard/iq/dataverse/search/SolrClientServiceTest.java b/src/test/java/edu/harvard/iq/dataverse/search/SolrClientServiceTest.java index 72eafcd763c..ebb29e688bd 100644 --- a/src/test/java/edu/harvard/iq/dataverse/search/SolrClientServiceTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/search/SolrClientServiceTest.java @@ -5,7 +5,7 @@ import edu.harvard.iq.dataverse.util.SystemConfig; import edu.harvard.iq.dataverse.util.testing.JvmSetting; import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings; -import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.impl.Http2SolrClient; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -39,7 +39,7 @@ void testInitWithDefaults() { clientService.init(); // then - HttpSolrClient client = (HttpSolrClient) clientService.getSolrClient(); + Http2SolrClient client = (Http2SolrClient) clientService.getSolrClient(); assertEquals(url, client.getBaseURL()); } @@ -55,7 +55,7 @@ void testInitWithConfig() { clientService.init(); // then - HttpSolrClient client = (HttpSolrClient) clientService.getSolrClient(); + Http2SolrClient client = (Http2SolrClient) clientService.getSolrClient(); assertEquals(url, client.getBaseURL()); } } \ No newline at end of file From 65a9871d138f25b6fa12078a1bfd3998658bb53c Mon Sep 17 00:00:00 2001 From: stevenferey Date: Wed, 27 Mar 2024 11:29:34 +0100 Subject: [PATCH 002/137] 10161 - adaptation of Java imports --- .../java/edu/harvard/iq/dataverse/search/IndexServiceBean.java | 1 + .../edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index f032142cf09..e12ee111a18 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -10,6 +10,7 @@ import edu.harvard.iq.dataverse.DatasetFieldConstant; import edu.harvard.iq.dataverse.DatasetFieldServiceBean; import edu.harvard.iq.dataverse.DatasetFieldType; +import edu.harvard.iq.dataverse.DatasetFieldValue; import edu.harvard.iq.dataverse.DatasetFieldValueValidator; import edu.harvard.iq.dataverse.DatasetLinkingServiceBean; import edu.harvard.iq.dataverse.DatasetServiceBean; diff --git a/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java index bd73f05fea7..cf85899121c 100644 --- a/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java @@ -1,7 +1,6 @@ package edu.harvard.iq.dataverse.search; import edu.harvard.iq.dataverse.ControlledVocabularyValue; -import edu.harvard.iq.dataverse.DOIServiceBean; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetField; import edu.harvard.iq.dataverse.DatasetFieldCompoundValue; From a0a7c8d05bf5fd0416a48a6052d87c987603312f Mon Sep 17 00:00:00 2001 From: Thibault Coupin Date: Mon, 3 Oct 2022 09:26:24 +0200 Subject: [PATCH 003/137] Harvest: map publisher tag to distributorName --- .../db/migration/V5.12.0.1__8739-publisher-during-harvesting.sql | 1 + 1 file changed, 1 insertion(+) create mode 100644 src/main/resources/db/migration/V5.12.0.1__8739-publisher-during-harvesting.sql diff --git a/src/main/resources/db/migration/V5.12.0.1__8739-publisher-during-harvesting.sql b/src/main/resources/db/migration/V5.12.0.1__8739-publisher-during-harvesting.sql new file mode 100644 index 00000000000..c4dbd901181 --- /dev/null +++ b/src/main/resources/db/migration/V5.12.0.1__8739-publisher-during-harvesting.sql @@ -0,0 +1 @@ +update foreignmetadatafieldmapping set datasetfieldname = 'distributorName' where foreignfieldxpath = ':publisher'; From d92d048354d19ca58665f0db6fd0cb673dd6f985 Mon Sep 17 00:00:00 2001 From: plecor <146710476+plecor@users.noreply.github.com> Date: Thu, 2 May 2024 13:40:00 +0200 Subject: [PATCH 004/137] Rename migration file --- ...vesting.sql => V6.2.0.2__8739-publisher-during-harvesting.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V5.12.0.1__8739-publisher-during-harvesting.sql => V6.2.0.2__8739-publisher-during-harvesting.sql} (100%) diff --git a/src/main/resources/db/migration/V5.12.0.1__8739-publisher-during-harvesting.sql b/src/main/resources/db/migration/V6.2.0.2__8739-publisher-during-harvesting.sql similarity index 100% rename from src/main/resources/db/migration/V5.12.0.1__8739-publisher-during-harvesting.sql rename to src/main/resources/db/migration/V6.2.0.2__8739-publisher-during-harvesting.sql From d6a6e56df7bae324e95c25e7a2b0de5f6d273aa1 Mon Sep 17 00:00:00 2001 From: plecor <146710476+plecor@users.noreply.github.com> Date: Tue, 14 May 2024 15:31:20 +0200 Subject: [PATCH 005/137] Add use case to HarvestingClientsIT --- .../edu/harvard/iq/dataverse/api/HarvestingClientsIT.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java index 340eab161bb..5020e37edb8 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/HarvestingClientsIT.java @@ -299,6 +299,11 @@ private void harvestingClientRun(boolean allowHarvestingMissingCVV) throws Inte } // verify count after collecting global ids assertEquals(expectedNumberOfSetsHarvested, jsonPath.getInt("data.total_count")); + + // ensure the publisher name is present in the harvested dataset citation + Response harvestedDataverse = given().get(ARCHIVE_URL + "/api/dataverses/1"); + String harvestedDataverseName = harvestedDataverse.getBody().jsonPath().getString("data.name"); + assertTrue(jsonPath.getString("data.items[0].citation").contains(harvestedDataverseName)); // Fail if it hasn't completed in maxWait seconds assertTrue(i < maxWait); From 64b69b94a7677d927c81b30f1a9cf43b470412eb Mon Sep 17 00:00:00 2001 From: plecor <146710476+plecor@users.noreply.github.com> Date: Tue, 14 May 2024 15:50:19 +0200 Subject: [PATCH 006/137] Add release note --- doc/release-notes/8739-publisher-during-harvesting.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/release-notes/8739-publisher-during-harvesting.md diff --git a/doc/release-notes/8739-publisher-during-harvesting.md b/doc/release-notes/8739-publisher-during-harvesting.md new file mode 100644 index 00000000000..602b2cf34d6 --- /dev/null +++ b/doc/release-notes/8739-publisher-during-harvesting.md @@ -0,0 +1 @@ +The publisher value of harvested datasets is now attributed to the dataset's distributor instead of its producer. This change affects all newly harvested datasets. For more information, see #8739 \ No newline at end of file From 5dfb01ef58f11dbac8ed4153e265e606b55fef0c Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 14 May 2024 13:55:24 -0400 Subject: [PATCH 007/137] add change in behavior to new harvesting client changlog #8739 --- doc/sphinx-guides/source/admin/harvestclients.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/sphinx-guides/source/admin/harvestclients.rst b/doc/sphinx-guides/source/admin/harvestclients.rst index 59fc4dc2c64..1e45fd06fc8 100644 --- a/doc/sphinx-guides/source/admin/harvestclients.rst +++ b/doc/sphinx-guides/source/admin/harvestclients.rst @@ -47,3 +47,8 @@ What if a Run Fails? Each harvesting client run logs a separate file per run to the app server's default logging directory (``/usr/local/payara6/glassfish/domains/domain1/logs/`` unless you've changed it). Look for filenames in the format ``harvest_TARGET_YYYY_MM_DD_timestamp.log`` to get a better idea of what's going wrong. Note that you'll want to run a minimum of Dataverse Software 4.6, optimally 4.18 or beyond, for the best OAI-PMH interoperability. + +Harvesting Client Changelog +--------------------------- + +- As of Dataverse 6.3, the publisher value of harvested datasets is now attributed to the dataset's distributor instead of its producer. This change affects all newly harvested datasets. For more information, see https://github.com/IQSS/dataverse/pull/9013 From 1eceaeef0ac2e2f435e8238c9804d67d4d636f4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20ROUCOU?= Date: Wed, 26 Jun 2024 18:15:02 +0200 Subject: [PATCH 008/137] fix compilation failure after merge --- .../java/edu/harvard/iq/dataverse/search/IndexServiceBean.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index 0c594653870..6e593eb223b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -25,6 +25,7 @@ import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.GlobalId; import edu.harvard.iq.dataverse.PermissionServiceBean; +import edu.harvard.iq.dataverse.Retention; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUserServiceBean; import edu.harvard.iq.dataverse.batch.util.LoggingUtil; From 0fd9fce1c9dfbbac9e69961c70fceba8eeef6a48 Mon Sep 17 00:00:00 2001 From: Dimitri Szabo <46443753+DS-INRA@users.noreply.github.com> Date: Wed, 3 Jul 2024 15:07:31 +0200 Subject: [PATCH 009/137] Update README.md Added sections, emojis and more links. Epanded the text --- README.md | 51 +++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 45 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 77720453d5f..60359e7c618 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,55 @@ Dataverse® =============== -Dataverse is an [open source][] software platform for sharing, finding, citing, and preserving research data (developed by the [Dataverse team](https://dataverse.org/about) at the [Institute for Quantitative Social Science](https://iq.harvard.edu/) and the [Dataverse community][]). +Welcome to Dataverseยฎ, the [open source][] software platform designed for sharing, finding, citing, and preserving research data. Developed by the Dataverse team at the [Institute for Quantitative Social Science](https://iq.harvard.edu/) and the [Dataverse community][], our platform is built to support researchers and institutions in sharing, finding, citing, and preserving research their data effectively. -[dataverse.org][] is our home on the web and shows a map of Dataverse installations around the world, a list of [features][], [integrations][] that have been made possible through [REST APIs][], our [project board][], our development [roadmap][], and more. +## โœ” Try Dataverse -We maintain a demo site at [demo.dataverse.org][] which you are welcome to use for testing and evaluating Dataverse. +We invite you to explore our demo site at [demo.dataverse.org][]. This site is ideal for testing and evaluating Dataverse in a risk-free environment. -To install Dataverse, please see our [Installation Guide][] which will prompt you to download our [latest release][]. Docker users should consult the [Container Guide][]. +## ๐ŸŒ Our Web Presence -To discuss Dataverse with the community, please join our [mailing list][], participate in a [community call][], chat with us at [chat.dataverse.org][], or attend our annual [Dataverse Community Meeting][]. +Visit [dataverse.org][], our home on the web, for a comprehensive overview of Dataverse. Here, you will find: -We love contributors! Please see our [Contributing Guide][] for ways you can help. +- An interactive map showcasing Dataverse installations worldwide. +- A detailed list of [features][]. +- Information on [integrations][] that have been made possible through our [REST APIs][]. +- Our [project board][] and development [roadmap][]. +- News, events, and more. + +## ๐Ÿ“ฅ Installation + +Ready to get started? Follow our [Installation Guide][] to download and install the latest release of Dataverse. + +If you are using Docker, please refer to our [Container Guide][] for detailed instructions. + +## ๐Ÿ˜ Community and Support + +Engage with the vibrant Dataverse community through various channels: + +- **[Mailing List][]**: Join the conversation on our [mailing list][]. +- **[Community Calls][]**: Participate in our regular [community calls][] to discuss new features, ask questions, and share your experiences. +- **[Chat][]**: Connect with us and other users in real-time at [chat.dataverse.org][]. +- **[Dataverse Community Meeting][]**: Attend our annual [Dataverse Community Meeting][] to network, learn, and collaborate with peers and experts. +- **[DataverseTV][]**: Watch the video content from the Dataverse community on [DataverseTV][] and on [Harvard's IQSS YouTube channel][]. + +## ๐Ÿง‘โ€๐Ÿ’ป๏ธ Contribute to Dataverse + +We love contributors! Whether you are a developer, researcher, or enthusiast, there are many ways you can help. + +Visit our [Contributing Guide][] to learn how you can get involved. + +Join us in building and enhancing Dataverse to make research data more accessible and impactful. Your support and participation are crucial to our success! + +## โš–๏ธ Legal Information Dataverse is a trademark of President and Fellows of Harvard College and is registered in the United States. +--- +For more detailed information, visit our website at [dataverse.org][]. + +Feel free to reach out with any questions or feedback. Happy researching! + [![Dataverse Project logo](src/main/webapp/resources/images/dataverseproject_logo.jpg "Dataverse Project")](http://dataverse.org) [![API Test Status](https://jenkins.dataverse.org/buildStatus/icon?job=IQSS-dataverse-develop&subject=API%20Test%20Status)](https://jenkins.dataverse.org/job/IQSS-dataverse-develop/) @@ -37,6 +72,10 @@ Dataverse is a trademark of President and Fellows of Harvard College and is regi [Contributing Guide]: CONTRIBUTING.md [mailing list]: https://groups.google.com/group/dataverse-community [community call]: https://dataverse.org/community-calls +[Chat]: https://chat.dataverse.org [chat.dataverse.org]: https://chat.dataverse.org [Dataverse Community Meeting]: https://dataverse.org/events [open source]: LICENSE.md +[community calls]: https://dataverse.org/community-calls +[DataverseTV]: https://dataverse.org/dataversetv +[Harvard's IQSS YouTube channel]: https://www.youtube.com/@iqssatharvarduniversity8672 From e3f47c5b9ce5dd79a6dc439289dfb7148ea45d47 Mon Sep 17 00:00:00 2001 From: Dimitri Szabo <46443753+DS-INRA@users.noreply.github.com> Date: Fri, 19 Jul 2024 16:27:50 +0200 Subject: [PATCH 010/137] Add logo and Table of contents --- README.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/README.md b/README.md index 60359e7c618..ef184e835f2 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,38 @@ Dataverse® =============== +
+ + +## Table of Contents + +
    +
  1. โ“ What is Dataverse?
    2. +
  2. โœ” Try Dataverse
    3. +
  3. ๐ŸŒ Our Web Presence
    4. +
  4. ๐Ÿ“ฅ Installation
    5. +
  5. ๐Ÿ˜ Community and Support
    6. +
  6. ๐Ÿง‘โ€๐Ÿ’ป๏ธ Contributing
    7. +
  7. โš–๏ธ Legal Information
    8. +
+ + + +## โ“ What is Dataverse? Welcome to Dataverseยฎ, the [open source][] software platform designed for sharing, finding, citing, and preserving research data. Developed by the Dataverse team at the [Institute for Quantitative Social Science](https://iq.harvard.edu/) and the [Dataverse community][], our platform is built to support researchers and institutions in sharing, finding, citing, and preserving research their data effectively. + + ## โœ” Try Dataverse We invite you to explore our demo site at [demo.dataverse.org][]. This site is ideal for testing and evaluating Dataverse in a risk-free environment. + + ## ๐ŸŒ Our Web Presence Visit [dataverse.org][], our home on the web, for a comprehensive overview of Dataverse. Here, you will find: @@ -17,12 +43,16 @@ Visit [dataverse.org][], our home on the web, for a comprehensive overview of Da - Our [project board][] and development [roadmap][]. - News, events, and more. + + ## ๐Ÿ“ฅ Installation Ready to get started? Follow our [Installation Guide][] to download and install the latest release of Dataverse. If you are using Docker, please refer to our [Container Guide][] for detailed instructions. + + ## ๐Ÿ˜ Community and Support Engage with the vibrant Dataverse community through various channels: @@ -33,6 +63,7 @@ Engage with the vibrant Dataverse community through various channels: - **[Dataverse Community Meeting][]**: Attend our annual [Dataverse Community Meeting][] to network, learn, and collaborate with peers and experts. - **[DataverseTV][]**: Watch the video content from the Dataverse community on [DataverseTV][] and on [Harvard's IQSS YouTube channel][]. + ## ๐Ÿง‘โ€๐Ÿ’ป๏ธ Contribute to Dataverse We love contributors! Whether you are a developer, researcher, or enthusiast, there are many ways you can help. @@ -41,6 +72,7 @@ Visit our [Contributing Guide][] to learn how you can get involved. Join us in building and enhancing Dataverse to make research data more accessible and impactful. Your support and participation are crucial to our success! + ## โš–๏ธ Legal Information Dataverse is a trademark of President and Fellows of Harvard College and is registered in the United States. From e8a9db28b45c1e55ed7fcd4afc07c7778597d253 Mon Sep 17 00:00:00 2001 From: Dimitri Szabo <46443753+DS-INRA@users.noreply.github.com> Date: Fri, 19 Jul 2024 23:48:49 +0200 Subject: [PATCH 011/137] Updated ToC --- README.md | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index ef184e835f2..54ad2f780ab 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,17 @@ Dataverse® =============== -
- + +![309827846-6c4d79e4-7be5-4102-88bd-dfa167dc79d3](https://github.com/user-attachments/assets/cd5d0e97-47cf-45be-9639-179839adafe0) ## Table of Contents -
    -
  1. โ“ What is Dataverse?
    2. -
  2. โœ” Try Dataverse
    3. -
  3. ๐ŸŒ Our Web Presence
    4. -
  4. ๐Ÿ“ฅ Installation
    5. -
  5. ๐Ÿ˜ Community and Support
    6. -
  6. ๐Ÿง‘โ€๐Ÿ’ป๏ธ Contributing
    7. -
  7. โš–๏ธ Legal Information
    8. -
+1. [โ“ What is Dataverse?](#what-is-dataverse) +2. [โœ” Try Dataverse](#try-dataverse) +3. [๐ŸŒ Our Web Presence](#our-web-presence) +4. [๐Ÿ“ฅ Installation](#installation) +5. [๐Ÿ˜ Community and Support](#community-and-support) +6. [๐Ÿง‘โ€๐Ÿ’ป๏ธ Contributing](#contributing) +7. [โš–๏ธ Legal Information](#legal-informations) From a013340b36ab8db916cbe8dc5eb81c23d88f01e4 Mon Sep 17 00:00:00 2001 From: Dimitri Szabo <46443753+DS-INRA@users.noreply.github.com> Date: Fri, 19 Jul 2024 23:54:33 +0200 Subject: [PATCH 012/137] Fix Logo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 54ad2f780ab..850a6c5eed1 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Dataverse® =============== -![309827846-6c4d79e4-7be5-4102-88bd-dfa167dc79d3](https://github.com/user-attachments/assets/cd5d0e97-47cf-45be-9639-179839adafe0) +![Dataverse-logo](https://github.com/IQSS/dataverse-frontend/assets/7512607/6c4d79e4-7be5-4102-88bd-dfa167dc79d3) ## Table of Contents From 022b862e04f714b8ff9aed2046cea2277107173f Mon Sep 17 00:00:00 2001 From: Ludovic DANIEL Date: Thu, 5 Sep 2024 15:30:28 +0200 Subject: [PATCH 013/137] Add missing imports after merge --- .../java/edu/harvard/iq/dataverse/search/IndexServiceBean.java | 1 + .../edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java | 1 + 2 files changed, 2 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index cfdb20c7e6b..168517c0ebf 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -16,6 +16,7 @@ import edu.harvard.iq.dataverse.DatasetServiceBean; import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.DatasetVersion.VersionState; +import edu.harvard.iq.dataverse.DatasetVersionServiceBean; import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.DataverseLinkingServiceBean; import edu.harvard.iq.dataverse.DataverseServiceBean; diff --git a/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java index f9373f7dd78..7af466021de 100644 --- a/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java @@ -9,6 +9,7 @@ import edu.harvard.iq.dataverse.DatasetFieldType; import edu.harvard.iq.dataverse.DatasetFieldValue; import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.DatasetVersionServiceBean; import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.Dataverse.DataverseType; import edu.harvard.iq.dataverse.DataverseServiceBean; From 490a6d219f4fceb9c8b6dd331506d210b7923594 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20ROUCOU?= Date: Fri, 25 Oct 2024 18:20:06 +0200 Subject: [PATCH 014/137] Activation via feature-flag --- .../search/AbstractSolrClientService.java | 5 +++ .../search/SolrClientIndexService.java | 14 ++++++-- .../dataverse/search/SolrClientService.java | 19 +++++----- .../iq/dataverse/settings/FeatureFlags.java | 8 ++++- .../search/SolrClientIndexServiceTest.java | 34 +++++++++++++++++- .../search/SolrClientServiceTest.java | 35 ++++++++++++------- 6 files changed, 89 insertions(+), 26 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/AbstractSolrClientService.java b/src/main/java/edu/harvard/iq/dataverse/search/AbstractSolrClientService.java index f36c4a9e591..1ae236d348f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/AbstractSolrClientService.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/AbstractSolrClientService.java @@ -9,6 +9,11 @@ import edu.harvard.iq.dataverse.util.SystemConfig; import jakarta.ejb.EJB; +/** + * Generics methods for Solr clients implementations + * + * @author jeromeroucou + */ public abstract class AbstractSolrClientService { private static final Logger logger = Logger.getLogger(AbstractSolrClientService.class.getCanonicalName()); diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrClientIndexService.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrClientIndexService.java index 6a98c7cd4a0..59281d5fa4e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrClientIndexService.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrClientIndexService.java @@ -3,7 +3,9 @@ import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient; import org.apache.solr.client.solrj.impl.Http2SolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import edu.harvard.iq.dataverse.settings.FeatureFlags; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import jakarta.ejb.Singleton; @@ -12,7 +14,7 @@ /** * Solr client to provide insert/update/delete operations. - * Don't use this service with queries to Solr, use SolrClientService instend. + * Don't use this service with queries to Solr, use {@link SolrClientService} instead. */ @Named @Singleton @@ -23,8 +25,14 @@ public class SolrClientIndexService extends AbstractSolrClientService { @PostConstruct public void init() { - solrClient = new ConcurrentUpdateHttp2SolrClient.Builder( - getSolrUrl(), new Http2SolrClient.Builder().build()).build(); + if (FeatureFlags.ENABLE_HTTP2_SOLR_CLIENT.enabled()) { + solrClient = new ConcurrentUpdateHttp2SolrClient.Builder( + getSolrUrl(), new Http2SolrClient.Builder().build()).build(); + } else { + // ConcurrentUpdateSolrClient seem to be more suitable, but + // actually only HttpSolrClient is used. + solrClient = new HttpSolrClient.Builder(getSolrUrl()).build(); + } } @PreDestroy diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrClientService.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrClientService.java index 60ab269455a..83f16e29af2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrClientService.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrClientService.java @@ -1,13 +1,10 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ package edu.harvard.iq.dataverse.search; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.impl.Http2SolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import edu.harvard.iq.dataverse.settings.FeatureFlags; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import jakarta.ejb.Singleton; @@ -18,9 +15,9 @@ * * @author landreev * - * This singleton is dedicated to initializing the Http2SolrClient used by the - * application to talk to the search engine, and serving it to all the other - * classes that need it. + * This singleton is dedicated to initializing the HttpSolrClient, or the Http2SolrClient + * (if feature-flag is enabled), used by the application to talk to the search engine, + * and serving it to all the other classes that need it. * This ensures that we are using one client only - as recommended by the * documentation. */ @@ -33,7 +30,11 @@ public class SolrClientService extends AbstractSolrClientService { @PostConstruct public void init() { - solrClient = new Http2SolrClient.Builder(getSolrUrl()).build(); + if (FeatureFlags.ENABLE_HTTP2_SOLR_CLIENT.enabled()) { + solrClient = new Http2SolrClient.Builder(getSolrUrl()).build(); + } else { + solrClient = new HttpSolrClient.Builder(getSolrUrl()).build(); + } } @PreDestroy diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java b/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java index 33e828e619d..97a8eae1d9c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java @@ -67,7 +67,13 @@ public enum FeatureFlags { * @since Dataverse 6.3 */ INDEX_HARVESTED_METADATA_SOURCE("index-harvested-metadata-source"), - + /** + * With this flag enabled, a the new Solr client Http2SolrClient is used in + * order to replace HttpSolrClient witch is deprecated since Solr 9. + * + * @apiNote Raise flag by setting "dataverse.feature.enable-http2-solr-client" + */ + ENABLE_HTTP2_SOLR_CLIENT("enable-http2-solr-client"), /** * Dataverse normally deletes all solr documents related to a dataset's files * when the dataset is reindexed. With this flag enabled, additional logic is diff --git a/src/test/java/edu/harvard/iq/dataverse/search/SolrClientIndexServiceTest.java b/src/test/java/edu/harvard/iq/dataverse/search/SolrClientIndexServiceTest.java index d6459626faf..09d9e096e55 100644 --- a/src/test/java/edu/harvard/iq/dataverse/search/SolrClientIndexServiceTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/search/SolrClientIndexServiceTest.java @@ -1,10 +1,14 @@ package edu.harvard.iq.dataverse.search; +import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.SystemConfig; +import edu.harvard.iq.dataverse.util.testing.JvmSetting; import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings; +import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -12,16 +16,21 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotNull; + @LocalJvmSettings @ExtendWith(MockitoExtension.class) class SolrClientIndexServiceTest { @Mock SettingsServiceBean settingsServiceBean; + @InjectMocks SystemConfig systemConfig; + SolrClientIndexService clientService = new SolrClientIndexService(); @BeforeEach @@ -31,12 +40,35 @@ void setUp() { @Test void testInitWithDefaults() { + // given + String url = "http://localhost:8983/solr/collection1"; + // when clientService.init(); // then - ConcurrentUpdateHttp2SolrClient client = (ConcurrentUpdateHttp2SolrClient) clientService.getSolrClient(); + SolrClient client = clientService.getSolrClient(); assertNotNull(client); + assertInstanceOf(HttpSolrClient.class, client); + assertEquals(url, clientService.getSolrUrl()); } + @Test + @JvmSetting(key = JvmSettings.SOLR_HOST, value = "foobar") + @JvmSetting(key = JvmSettings.SOLR_PORT, value = "1234") + @JvmSetting(key = JvmSettings.SOLR_CORE, value = "test") + @JvmSetting(key = JvmSettings.FEATURE_FLAG, value = "on", varArgs = "enable-http2-solr-client") + void testInitWithConfig() { + // given + String url = "http://foobar:1234/solr/test"; + + // when + clientService.init(); + + // then + SolrClient client = clientService.getSolrClient(); + assertNotNull(client); + assertInstanceOf(ConcurrentUpdateHttp2SolrClient.class, client); + assertEquals(url, clientService.getSolrUrl()); + } } \ No newline at end of file diff --git a/src/test/java/edu/harvard/iq/dataverse/search/SolrClientServiceTest.java b/src/test/java/edu/harvard/iq/dataverse/search/SolrClientServiceTest.java index ebb29e688bd..351d553f203 100644 --- a/src/test/java/edu/harvard/iq/dataverse/search/SolrClientServiceTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/search/SolrClientServiceTest.java @@ -5,7 +5,10 @@ import edu.harvard.iq.dataverse.util.SystemConfig; import edu.harvard.iq.dataverse.util.testing.JvmSetting; import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings; + +import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.impl.Http2SolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -14,48 +17,56 @@ import org.mockito.junit.jupiter.MockitoExtension; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; + @LocalJvmSettings @ExtendWith(MockitoExtension.class) class SolrClientServiceTest { - + @Mock SettingsServiceBean settingsServiceBean; @InjectMocks SystemConfig systemConfig; SolrClientService clientService = new SolrClientService(); - + @BeforeEach void setUp() { clientService.systemConfig = systemConfig; } - + @Test void testInitWithDefaults() { // given String url = "http://localhost:8983/solr/collection1"; - + // when clientService.init(); - + // then - Http2SolrClient client = (Http2SolrClient) clientService.getSolrClient(); - assertEquals(url, client.getBaseURL()); + SolrClient client = clientService.getSolrClient(); + assertNotNull(client); + assertInstanceOf(HttpSolrClient.class, client); + assertEquals(url, ((HttpSolrClient) client).getBaseURL()); } - + @Test @JvmSetting(key = JvmSettings.SOLR_HOST, value = "foobar") @JvmSetting(key = JvmSettings.SOLR_PORT, value = "1234") @JvmSetting(key = JvmSettings.SOLR_CORE, value = "test") + @JvmSetting(key = JvmSettings.FEATURE_FLAG, value = "on", varArgs = "enable-http2-solr-client") void testInitWithConfig() { // given String url = "http://foobar:1234/solr/test"; - + // when clientService.init(); - + // then - Http2SolrClient client = (Http2SolrClient) clientService.getSolrClient(); - assertEquals(url, client.getBaseURL()); + SolrClient client = clientService.getSolrClient(); + assertNotNull(client); + assertInstanceOf(Http2SolrClient.class, client); + assertEquals(url, clientService.getSolrUrl()); } } \ No newline at end of file From 728efffc814bf04a94e6350ee14b7ec0a3889576 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20ROUCOU?= Date: Fri, 25 Oct 2024 18:21:04 +0200 Subject: [PATCH 015/137] Documentations --- doc/release-notes/10241-new-solr-client.md | 9 +++++++++ doc/sphinx-guides/source/installation/config.rst | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 doc/release-notes/10241-new-solr-client.md diff --git a/doc/release-notes/10241-new-solr-client.md b/doc/release-notes/10241-new-solr-client.md new file mode 100644 index 00000000000..211ad9d1ec9 --- /dev/null +++ b/doc/release-notes/10241-new-solr-client.md @@ -0,0 +1,9 @@ +[HttpSolrClient](https://solr.apache.org/docs/9_3_0/solrj/org/apache/solr/client/solrj/impl/HttpSolrClient.html) is deprecated as of Solr 9, and which will be removed in a future major release of Solr. It's recommended to use [Http2SolrClient](https://solr.apache.org/docs/9_3_0/solrj/org/apache/solr/client/solrj/impl/Http2SolrClient.html) instead. + +[Solr documentation](https://solr.apache.org/guide/solr/latest/deployment-guide/solrj.html#types-of-solrclients) describe it as a _async, non-blocking and general-purpose client that leverage HTTP/2 using the Jetty Http library_. + +With Solr 9.3.0, the Http2SolrClient is indicate as experimental. But since the 9.6 version of Solr, this mention is no longer maintained. + +For the time being, its activation is therefore conditional on the use of a [feature-flag](https://dataverse-guide--10241.org.readthedocs.build/en/10241/installation/config.html#feature-flags). Activation also enables ConcurrentUpdateHttp2SolrClient, which is supposed to be more efficient for indexing. + +For more information, see issue [#10161](https://github.com/IQSS/dataverse/issues/10161) and pull request [#10241](https://github.com/IQSS/dataverse/pull/10241) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index e98ed8f5189..9823e3b6432 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -3345,8 +3345,8 @@ please find all known feature flags below. Any of these flags can be activated u * - add-publicobject-solr-field - Adds an extra boolean field `PublicObject_b:true` for public content (published Collections, Datasets and Files). Once reindexed with these fields, we can rely on it to remove a very expensive Solr join on all such documents in Solr queries, significantly improving overall performance (by enabling the feature flag above, `avoid-expensive-solr-join`). These two flags are separate so that an instance can reindex their holdings before enabling the optimization in searches, thus avoiding having their public objects temporarily disappear from search results while the reindexing is in progress. - ``Off`` - * - reduce-solr-deletes - - Avoids deleting and recreating solr documents for dataset files when reindexing. + * - enable-http2-solr-client + - Enable a new Solr client ``Http2SolrClient`` instead of ``HttpSolrClient`` witch is deprecated since Solr 9. Also enables use of ``ConcurrentUpdateHttp2SolrClient``, recommended to send concurrent updates to Solr. More informations about this Solr clients on `Solr documentation `_. - ``Off`` * - reduce-solr-deletes - Avoids deleting and recreating solr documents for dataset files when reindexing. From 3adb19419792d9304a51e75e76619305b568aa3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20ROUCOU?= Date: Fri, 15 Nov 2024 17:51:36 +0100 Subject: [PATCH 016/137] revert feature-flag --- .../search/SolrClientIndexService.java | 17 ++++++----------- .../iq/dataverse/search/SolrClientService.java | 14 ++++---------- .../iq/dataverse/settings/FeatureFlags.java | 7 ------- .../search/SolrClientIndexServiceTest.java | 4 +--- .../dataverse/search/SolrClientServiceTest.java | 6 ++---- 5 files changed, 13 insertions(+), 35 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrClientIndexService.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrClientIndexService.java index 59281d5fa4e..0b7f1aae798 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrClientIndexService.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrClientIndexService.java @@ -1,16 +1,15 @@ package edu.harvard.iq.dataverse.search; +import java.util.logging.Logger; + import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient; import org.apache.solr.client.solrj.impl.Http2SolrClient; -import org.apache.solr.client.solrj.impl.HttpSolrClient; -import edu.harvard.iq.dataverse.settings.FeatureFlags; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import jakarta.ejb.Singleton; import jakarta.inject.Named; -import java.util.logging.Logger; /** * Solr client to provide insert/update/delete operations. @@ -19,20 +18,15 @@ @Named @Singleton public class SolrClientIndexService extends AbstractSolrClientService { + private static final Logger logger = Logger.getLogger(SolrClientIndexService.class.getCanonicalName()); private SolrClient solrClient; @PostConstruct public void init() { - if (FeatureFlags.ENABLE_HTTP2_SOLR_CLIENT.enabled()) { - solrClient = new ConcurrentUpdateHttp2SolrClient.Builder( - getSolrUrl(), new Http2SolrClient.Builder().build()).build(); - } else { - // ConcurrentUpdateSolrClient seem to be more suitable, but - // actually only HttpSolrClient is used. - solrClient = new HttpSolrClient.Builder(getSolrUrl()).build(); - } + solrClient = new ConcurrentUpdateHttp2SolrClient.Builder( + getSolrUrl(), new Http2SolrClient.Builder().build()).build(); } @PreDestroy @@ -51,4 +45,5 @@ public SolrClient getSolrClient() { public void setSolrClient(SolrClient solrClient) { this.solrClient = solrClient; } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrClientService.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrClientService.java index 83f16e29af2..f9d94b8c6d3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrClientService.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrClientService.java @@ -2,9 +2,7 @@ import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.impl.Http2SolrClient; -import org.apache.solr.client.solrj.impl.HttpSolrClient; -import edu.harvard.iq.dataverse.settings.FeatureFlags; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import jakarta.ejb.Singleton; @@ -15,9 +13,9 @@ * * @author landreev * - * This singleton is dedicated to initializing the HttpSolrClient, or the Http2SolrClient - * (if feature-flag is enabled), used by the application to talk to the search engine, - * and serving it to all the other classes that need it. + * This singleton is dedicated to initializing the Http2SolrClient, used by + * the application to talk to the search engine, and serving it to all the + * other classes that need it. * This ensures that we are using one client only - as recommended by the * documentation. */ @@ -30,11 +28,7 @@ public class SolrClientService extends AbstractSolrClientService { @PostConstruct public void init() { - if (FeatureFlags.ENABLE_HTTP2_SOLR_CLIENT.enabled()) { - solrClient = new Http2SolrClient.Builder(getSolrUrl()).build(); - } else { - solrClient = new HttpSolrClient.Builder(getSolrUrl()).build(); - } + solrClient = new Http2SolrClient.Builder(getSolrUrl()).build(); } @PreDestroy diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java b/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java index 97a8eae1d9c..24b41c623b6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java @@ -67,13 +67,6 @@ public enum FeatureFlags { * @since Dataverse 6.3 */ INDEX_HARVESTED_METADATA_SOURCE("index-harvested-metadata-source"), - /** - * With this flag enabled, a the new Solr client Http2SolrClient is used in - * order to replace HttpSolrClient witch is deprecated since Solr 9. - * - * @apiNote Raise flag by setting "dataverse.feature.enable-http2-solr-client" - */ - ENABLE_HTTP2_SOLR_CLIENT("enable-http2-solr-client"), /** * Dataverse normally deletes all solr documents related to a dataset's files * when the dataset is reindexed. With this flag enabled, additional logic is diff --git a/src/test/java/edu/harvard/iq/dataverse/search/SolrClientIndexServiceTest.java b/src/test/java/edu/harvard/iq/dataverse/search/SolrClientIndexServiceTest.java index 09d9e096e55..d3d68fa5f6a 100644 --- a/src/test/java/edu/harvard/iq/dataverse/search/SolrClientIndexServiceTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/search/SolrClientIndexServiceTest.java @@ -8,7 +8,6 @@ import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient; -import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -49,7 +48,7 @@ void testInitWithDefaults() { // then SolrClient client = clientService.getSolrClient(); assertNotNull(client); - assertInstanceOf(HttpSolrClient.class, client); + assertInstanceOf(ConcurrentUpdateHttp2SolrClient.class, client); assertEquals(url, clientService.getSolrUrl()); } @@ -57,7 +56,6 @@ void testInitWithDefaults() { @JvmSetting(key = JvmSettings.SOLR_HOST, value = "foobar") @JvmSetting(key = JvmSettings.SOLR_PORT, value = "1234") @JvmSetting(key = JvmSettings.SOLR_CORE, value = "test") - @JvmSetting(key = JvmSettings.FEATURE_FLAG, value = "on", varArgs = "enable-http2-solr-client") void testInitWithConfig() { // given String url = "http://foobar:1234/solr/test"; diff --git a/src/test/java/edu/harvard/iq/dataverse/search/SolrClientServiceTest.java b/src/test/java/edu/harvard/iq/dataverse/search/SolrClientServiceTest.java index 351d553f203..13cea4151ff 100644 --- a/src/test/java/edu/harvard/iq/dataverse/search/SolrClientServiceTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/search/SolrClientServiceTest.java @@ -8,7 +8,6 @@ import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.impl.Http2SolrClient; -import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -47,15 +46,14 @@ void testInitWithDefaults() { // then SolrClient client = clientService.getSolrClient(); assertNotNull(client); - assertInstanceOf(HttpSolrClient.class, client); - assertEquals(url, ((HttpSolrClient) client).getBaseURL()); + assertInstanceOf(Http2SolrClient.class, client); + assertEquals(url, clientService.getSolrUrl()); } @Test @JvmSetting(key = JvmSettings.SOLR_HOST, value = "foobar") @JvmSetting(key = JvmSettings.SOLR_PORT, value = "1234") @JvmSetting(key = JvmSettings.SOLR_CORE, value = "test") - @JvmSetting(key = JvmSettings.FEATURE_FLAG, value = "on", varArgs = "enable-http2-solr-client") void testInitWithConfig() { // given String url = "http://foobar:1234/solr/test"; From 6a05ecdcbbccae3fcc54711ba5a7b76dbd44049d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20ROUCOU?= Date: Fri, 15 Nov 2024 18:05:42 +0100 Subject: [PATCH 017/137] revert feature flag documentation --- doc/release-notes/10241-new-solr-client.md | 6 +++--- doc/sphinx-guides/source/installation/config.rst | 3 --- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/doc/release-notes/10241-new-solr-client.md b/doc/release-notes/10241-new-solr-client.md index 211ad9d1ec9..67ccdd4f184 100644 --- a/doc/release-notes/10241-new-solr-client.md +++ b/doc/release-notes/10241-new-solr-client.md @@ -1,9 +1,9 @@ -[HttpSolrClient](https://solr.apache.org/docs/9_3_0/solrj/org/apache/solr/client/solrj/impl/HttpSolrClient.html) is deprecated as of Solr 9, and which will be removed in a future major release of Solr. It's recommended to use [Http2SolrClient](https://solr.apache.org/docs/9_3_0/solrj/org/apache/solr/client/solrj/impl/Http2SolrClient.html) instead. +[HttpSolrClient](https://solr.apache.org/docs/9_4_1/solrj/org/apache/solr/client/solrj/impl/HttpSolrClient.html) is deprecated as of Solr 9, and which will be removed in a future major release of Solr. It's recommended to use [Http2SolrClient](https://solr.apache.org/docs/9_4_1/solrj/org/apache/solr/client/solrj/impl/Http2SolrClient.html) instead. [Solr documentation](https://solr.apache.org/guide/solr/latest/deployment-guide/solrj.html#types-of-solrclients) describe it as a _async, non-blocking and general-purpose client that leverage HTTP/2 using the Jetty Http library_. -With Solr 9.3.0, the Http2SolrClient is indicate as experimental. But since the 9.6 version of Solr, this mention is no longer maintained. +With Solr 9.4.1, the Http2SolrClient is indicate as experimental. But since the 9.6 version of Solr, this mention is no longer maintained. -For the time being, its activation is therefore conditional on the use of a [feature-flag](https://dataverse-guide--10241.org.readthedocs.build/en/10241/installation/config.html#feature-flags). Activation also enables ConcurrentUpdateHttp2SolrClient, which is supposed to be more efficient for indexing. +The ConcurrentUpdateHttp2SolrClient is now also used in some cases, which is supposed to be more efficient for indexing. For more information, see issue [#10161](https://github.com/IQSS/dataverse/issues/10161) and pull request [#10241](https://github.com/IQSS/dataverse/pull/10241) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 9823e3b6432..b4705aa6521 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -3345,9 +3345,6 @@ please find all known feature flags below. Any of these flags can be activated u * - add-publicobject-solr-field - Adds an extra boolean field `PublicObject_b:true` for public content (published Collections, Datasets and Files). Once reindexed with these fields, we can rely on it to remove a very expensive Solr join on all such documents in Solr queries, significantly improving overall performance (by enabling the feature flag above, `avoid-expensive-solr-join`). These two flags are separate so that an instance can reindex their holdings before enabling the optimization in searches, thus avoiding having their public objects temporarily disappear from search results while the reindexing is in progress. - ``Off`` - * - enable-http2-solr-client - - Enable a new Solr client ``Http2SolrClient`` instead of ``HttpSolrClient`` witch is deprecated since Solr 9. Also enables use of ``ConcurrentUpdateHttp2SolrClient``, recommended to send concurrent updates to Solr. More informations about this Solr clients on `Solr documentation `_. - - ``Off`` * - reduce-solr-deletes - Avoids deleting and recreating solr documents for dataset files when reindexing. - ``Off`` From 09fd6c1bd75d0d25e67f730342f98a897579f954 Mon Sep 17 00:00:00 2001 From: Pierre Le Corre <146710476+plecor@users.noreply.github.com> Date: Tue, 3 Dec 2024 16:44:35 +0100 Subject: [PATCH 018/137] Update doc/sphinx-guides/source/admin/harvestclients.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/admin/harvestclients.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/admin/harvestclients.rst b/doc/sphinx-guides/source/admin/harvestclients.rst index 1e45fd06fc8..3c4c0957c2b 100644 --- a/doc/sphinx-guides/source/admin/harvestclients.rst +++ b/doc/sphinx-guides/source/admin/harvestclients.rst @@ -51,4 +51,4 @@ Note that you'll want to run a minimum of Dataverse Software 4.6, optimally 4.18 Harvesting Client Changelog --------------------------- -- As of Dataverse 6.3, the publisher value of harvested datasets is now attributed to the dataset's distributor instead of its producer. This change affects all newly harvested datasets. For more information, see https://github.com/IQSS/dataverse/pull/9013 +- As of Dataverse 6.6, the publisher value of harvested datasets is now attributed to the dataset's distributor instead of its producer. This change affects all newly harvested datasets. For more information, see https://github.com/IQSS/dataverse/pull/9013 From 8b273e8bedebd2930e687a1d783dbea8739adf82 Mon Sep 17 00:00:00 2001 From: Pierre Le Corre <146710476+plecor@users.noreply.github.com> Date: Tue, 3 Dec 2024 16:44:58 +0100 Subject: [PATCH 019/137] Update doc/release-notes/8739-publisher-during-harvesting.md Co-authored-by: Philip Durbin --- doc/release-notes/8739-publisher-during-harvesting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/8739-publisher-during-harvesting.md b/doc/release-notes/8739-publisher-during-harvesting.md index 602b2cf34d6..05dcc53ac7d 100644 --- a/doc/release-notes/8739-publisher-during-harvesting.md +++ b/doc/release-notes/8739-publisher-during-harvesting.md @@ -1 +1 @@ -The publisher value of harvested datasets is now attributed to the dataset's distributor instead of its producer. This change affects all newly harvested datasets. For more information, see #8739 \ No newline at end of file +The publisher value of harvested datasets is now attributed to the dataset's distributor instead of its producer. This change affects all newly harvested datasets. For more information, see [the guides](https://dataverse-guide--9013.org.readthedocs.build/en/9013/admin/harvestclients.html#harvesting-client-changelog), #8739, and #9013. \ No newline at end of file From bc33fbfa2cc52353f7a0da0c58a6af0dcb87216f Mon Sep 17 00:00:00 2001 From: Pierre Le Corre <146710476+plecor@users.noreply.github.com> Date: Tue, 3 Dec 2024 16:47:57 +0100 Subject: [PATCH 020/137] Rename migration script --- ...vesting.sql => V6.4.0.1__8739-publisher-during-harvesting.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V6.2.0.2__8739-publisher-during-harvesting.sql => V6.4.0.1__8739-publisher-during-harvesting.sql} (100%) diff --git a/src/main/resources/db/migration/V6.2.0.2__8739-publisher-during-harvesting.sql b/src/main/resources/db/migration/V6.4.0.1__8739-publisher-during-harvesting.sql similarity index 100% rename from src/main/resources/db/migration/V6.2.0.2__8739-publisher-during-harvesting.sql rename to src/main/resources/db/migration/V6.4.0.1__8739-publisher-during-harvesting.sql From 8503729634b35b885a3047867e8c2e361735fb65 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Tue, 3 Dec 2024 14:43:33 -0500 Subject: [PATCH 021/137] sql script/flyway to fix file access requests --- ...10714-access-requests-missing-since-upgrade-v6-0.md | 6 ++++++ src/main/resources/db/migration/V6.4.0.4.sql | 10 ++++++++++ 2 files changed, 16 insertions(+) create mode 100644 doc/release-notes/10714-access-requests-missing-since-upgrade-v6-0.md create mode 100644 src/main/resources/db/migration/V6.4.0.4.sql diff --git a/doc/release-notes/10714-access-requests-missing-since-upgrade-v6-0.md b/doc/release-notes/10714-access-requests-missing-since-upgrade-v6-0.md new file mode 100644 index 00000000000..a220c150791 --- /dev/null +++ b/doc/release-notes/10714-access-requests-missing-since-upgrade-v6-0.md @@ -0,0 +1,6 @@ +### Flyway Script added to Fix File Access Requests when upgrading from Dataverse 6.0 + +Database update script added to prevent duplicate keys when upgrading from V6.0 +This script will delete access requests made after the initial request and will set the initial request to "Created" + +See: https://github.com/IQSS/dataverse/issues/10714 diff --git a/src/main/resources/db/migration/V6.4.0.4.sql b/src/main/resources/db/migration/V6.4.0.4.sql new file mode 100644 index 00000000000..804ce3c1ea8 --- /dev/null +++ b/src/main/resources/db/migration/V6.4.0.4.sql @@ -0,0 +1,10 @@ +-- Fixes File Access Requests when upgrading from Dataverse 6.0 +-- See: https://github.com/IQSS/dataverse/issues/10714 +DELETE FROM fileaccessrequests +WHERE creation_time <> (SELECT MIN(creation_time) + FROM fileaccessrequests far2 + WHERE far2.datafile_id = fileaccessrequests.datafile_id + AND far2.authenticated_user_id = fileaccessrequests.authenticated_user_id + AND far2.request_state is NULL); + +UPDATE fileaccessrequests SET request_state='CREATED' WHERE request_state is NULL; From 409822ee4399b0d9a8e6f1e9627b01dc237f930a Mon Sep 17 00:00:00 2001 From: stevenferey Date: Wed, 4 Dec 2024 15:04:24 +0100 Subject: [PATCH 022/137] Add Move Dataverse Collection --- .../10304-add-move-dataverse-collection.md | 5 + ...age.java => DashboardDatasetmovePage.java} | 23 +-- .../dashboard/DashboardDataversemovePage.java | 175 ++++++++++++++++++ .../command/impl/MoveDatasetCommand.java | 8 +- src/main/java/propertyFiles/Bundle.properties | 72 ++++--- ...move.xhtml => dashboard-datasetmove.xhtml} | 46 ++--- src/main/webapp/dashboard-dataversemove.xhtml | 140 ++++++++++++++ src/main/webapp/dashboard.xhtml | 17 +- 8 files changed, 413 insertions(+), 73 deletions(-) create mode 100644 doc/release-notes/10304-add-move-dataverse-collection.md rename src/main/java/edu/harvard/iq/dataverse/dashboard/{DashboardDatamovePage.java => DashboardDatasetmovePage.java} (90%) create mode 100644 src/main/java/edu/harvard/iq/dataverse/dashboard/DashboardDataversemovePage.java rename src/main/webapp/{dashboard-datamove.xhtml => dashboard-datasetmove.xhtml} (86%) create mode 100644 src/main/webapp/dashboard-dataversemove.xhtml diff --git a/doc/release-notes/10304-add-move-dataverse-collection.md b/doc/release-notes/10304-add-move-dataverse-collection.md new file mode 100644 index 00000000000..75237743ed7 --- /dev/null +++ b/doc/release-notes/10304-add-move-dataverse-collection.md @@ -0,0 +1,5 @@ +### Move a collection from the dashboard + +It is possible for a Dataverse administrator to move a collection from the Dataverse dashboard. + +For more information, see #10304. \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/dashboard/DashboardDatamovePage.java b/src/main/java/edu/harvard/iq/dataverse/dashboard/DashboardDatasetmovePage.java similarity index 90% rename from src/main/java/edu/harvard/iq/dataverse/dashboard/DashboardDatamovePage.java rename to src/main/java/edu/harvard/iq/dataverse/dashboard/DashboardDatasetmovePage.java index 6fc80312bf5..e3c1a2ad84e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dashboard/DashboardDatamovePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/dashboard/DashboardDatasetmovePage.java @@ -33,8 +33,8 @@ import jakarta.servlet.http.HttpServletRequest; @ViewScoped -@Named("DashboardDatamovePage") -public class DashboardDatamovePage implements java.io.Serializable { +@Named("DashboardDatasetmovePage") +public class DashboardDatasetmovePage implements java.io.Serializable { @Inject DataverseSession session; @@ -53,7 +53,7 @@ public class DashboardDatamovePage implements java.io.Serializable { @PersistenceContext(unitName = "VDCNet-ejbPU") private EntityManager em; - private static final Logger logger = Logger.getLogger(DashboardDatamovePage.class.getCanonicalName()); + private static final Logger logger = Logger.getLogger(DashboardDatasetmovePage.class.getCanonicalName()); private AuthenticatedUser authUser = null; @@ -122,8 +122,8 @@ public String init() { FacesContext.getCurrentInstance().addMessage(null, new FacesMessage(FacesMessage.SEVERITY_INFO, - BundleUtil.getStringFromBundle("dashboard.card.datamove.manage"), - BundleUtil.getStringFromBundle("dashboard.card.datamove.message", Arrays.asList(settingsWrapper.getGuidesBaseUrl(), settingsWrapper.getGuidesVersion())))); + BundleUtil.getStringFromBundle("dashboard.card.datasetmove.manage"), + BundleUtil.getStringFromBundle("dashboard.card.datasetmove.message", Arrays.asList(settingsWrapper.getGuidesBaseUrl(), settingsWrapper.getGuidesVersion())))); return null; } @@ -163,7 +163,7 @@ public void move(){ logger.info("Moved " + dsPersistentId + " from " + srcAlias + " to " + dstAlias); - JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dashboard.card.datamove.message.success", arguments)); + JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dashboard.card.datasetmove.message.success", arguments)); } catch (CommandException e) { logger.log(Level.SEVERE,"Unable to move "+ dsPersistentId + " from " + srcAlias + " to " + dstAlias, e); @@ -172,23 +172,18 @@ public void move(){ String guidesBaseUrl = settingsWrapper.getGuidesBaseUrl(); String version = settingsWrapper.getGuidesVersion(); // Suggest using the API to force the move. - arguments.add(BundleUtil.getStringFromBundle("dashboard.card.datamove.dataset.command.error.unforced.suggestForce", Arrays.asList(guidesBaseUrl, version))); + arguments.add(BundleUtil.getStringFromBundle("dashboard.card.datasetmove.dataset.command.error.unforced.suggestForce", Arrays.asList(guidesBaseUrl, version))); } else { String emptyStringNoDetails = ""; arguments.add(emptyStringNoDetails); } FacesContext.getCurrentInstance().addMessage(null, new FacesMessage(FacesMessage.SEVERITY_ERROR, - BundleUtil.getStringFromBundle("dashboard.card.datamove.message.failure.summary"), - BundleUtil.getStringFromBundle("dashboard.card.datamove.message.failure.details", arguments))); + BundleUtil.getStringFromBundle("dashboard.card.datasetmove.message.failure.summary"), + BundleUtil.getStringFromBundle("dashboard.card.datasetmove.message.failure.details", arguments))); } } - public String getDataverseCount() { - long count = em.createQuery("SELECT count(dv) FROM Dataverse dv", Long.class).getSingleResult(); - return NumberFormat.getInstance().format(count); - } - public String getDatasetCount() { long count = em.createQuery("SELECT count(ds) FROM Dataset ds", Long.class).getSingleResult(); return NumberFormat.getInstance().format(count); diff --git a/src/main/java/edu/harvard/iq/dataverse/dashboard/DashboardDataversemovePage.java b/src/main/java/edu/harvard/iq/dataverse/dashboard/DashboardDataversemovePage.java new file mode 100644 index 00000000000..df821dc953a --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/dashboard/DashboardDataversemovePage.java @@ -0,0 +1,175 @@ +package edu.harvard.iq.dataverse.dashboard; + +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetServiceBean; +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.DataverseServiceBean; +import edu.harvard.iq.dataverse.DataverseSession; +import edu.harvard.iq.dataverse.EjbDataverseEngine; +import edu.harvard.iq.dataverse.PermissionsWrapper; +import edu.harvard.iq.dataverse.SettingsWrapper; +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import edu.harvard.iq.dataverse.engine.command.exception.UnforcedCommandException; +import edu.harvard.iq.dataverse.engine.command.impl.MoveDatasetCommand; +import edu.harvard.iq.dataverse.engine.command.impl.MoveDataverseCommand; +import edu.harvard.iq.dataverse.util.BundleUtil; +import edu.harvard.iq.dataverse.util.JsfHelper; +import java.text.NumberFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; +import jakarta.ejb.EJB; +import jakarta.faces.application.FacesMessage; +import jakarta.faces.component.UIInput; +import jakarta.faces.context.FacesContext; +import jakarta.faces.view.ViewScoped; +import jakarta.inject.Inject; +import jakarta.inject.Named; +import jakarta.persistence.EntityManager; +import jakarta.persistence.PersistenceContext; +import jakarta.servlet.http.HttpServletRequest; + +@ViewScoped +@Named("DashboardDataversemovePage") +public class DashboardDataversemovePage implements java.io.Serializable { + + @Inject + DataverseSession session; + @Inject + PermissionsWrapper permissionsWrapper; + @EJB + EjbDataverseEngine commandEngine; + + //@EJB + //DatasetServiceBean datasetService; + @EJB + DataverseServiceBean dataverseService; + @Inject + SettingsWrapper settingsWrapper; + + @PersistenceContext(unitName = "VDCNet-ejbPU") + private EntityManager em; + + private static final Logger logger = Logger.getLogger(DashboardDataversemovePage.class.getCanonicalName()); + + private AuthenticatedUser authUser = null; + + // source dataverse + + public UIInput getSelectedSourceDataverseMenu() { + return selectedSourceDataverseMenu; + } + + public void setSelectedSourceDataverseMenu(UIInput selectedSourceDataverseMenu) { + this.selectedSourceDataverseMenu = selectedSourceDataverseMenu; + } + + UIInput selectedSourceDataverseMenu; + + public Dataverse getSelectedSourceDataverse() { + return selectedSourceDataverse; + } + + public void setSelectedSourceDataverse(Dataverse selectedSourceDataverse) { + this.selectedSourceDataverse = selectedSourceDataverse; + } + + private Dataverse selectedSourceDataverse; + + // destination dataverse + + public UIInput getSelectedDataverseMenu() { + return selectedDataverseMenu; + } + + public void setSelectedDataverseMenu(UIInput selectedDataverseMenu) { + this.selectedDataverseMenu = selectedDataverseMenu; + } + + UIInput selectedDataverseMenu; + + public Dataverse getSelectedDestinationDataverse() { + return selectedDestinationDataverse; + } + + public void setSelectedDestinationDataverse(Dataverse selectedDestinationDataverse) { + this.selectedDestinationDataverse = selectedDestinationDataverse; + } + + private Dataverse selectedDestinationDataverse; + + public List completeSelectedDataverse(String query) { + return dataverseService.filterByAliasQuery(query); + } + + public String init() { + + if ((session.getUser() != null) && (session.getUser().isAuthenticated()) && (session.getUser().isSuperuser())) { + authUser = (AuthenticatedUser) session.getUser(); + // initialize components, if any need it + } else { + return permissionsWrapper.notAuthorized(); + // redirect to login OR give some type of โ€˜you must be logged in' message + } + + FacesContext.getCurrentInstance().addMessage(null, + new FacesMessage(FacesMessage.SEVERITY_INFO, + BundleUtil.getStringFromBundle("dashboard.card.dataversemove.message.summary"), + BundleUtil.getStringFromBundle("dashboard.card.dataversemove.message.detail", Arrays.asList(settingsWrapper.getGuidesBaseUrl(), settingsWrapper.getGuidesVersion())))); + return null; + } + + public void move(){ + Dataverse dvSource = selectedSourceDataverse; + String srcAlias = dvSource!=null?dvSource.getAlias():null; + + Dataverse target = selectedDestinationDataverse; + String dstAlias = target!=null?target.getAlias():null; + + if (dvSource == null || target == null) { + // Move only works if both inputs are correct + // But if these inputs are required, we should never get here + // Since we never get here, we aren't bothering to move this English to the bundle. + FacesContext.getCurrentInstance().addMessage(null, + new FacesMessage("Please specify all fields")); + return; + } + + // construct arguments for message + List arguments = new ArrayList<>(); + arguments.add(dvSource!=null?dvSource.getName():"-"); + arguments.add(target!=null?target.getName():"-"); + + // copied logic from Dataverse API move + //Command requires Super user - it will be tested by the command + try { + HttpServletRequest httpServletRequest = (HttpServletRequest) FacesContext.getCurrentInstance().getExternalContext().getRequest(); + DataverseRequest dataverseRequest = new DataverseRequest(authUser, httpServletRequest); + commandEngine.submit(new MoveDataverseCommand( + dataverseRequest, dvSource, target, false + )); + + logger.info("Moved " + srcAlias + " to " + dstAlias); + + JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dashboard.card.dataversemove.message.success", arguments)); + } + catch (CommandException e) { + logger.log(Level.SEVERE,"Unable to move "+ srcAlias + " to " + dstAlias, e); + arguments.add(e.getLocalizedMessage()); + FacesContext.getCurrentInstance().addMessage(null, + new FacesMessage(FacesMessage.SEVERITY_ERROR, + BundleUtil.getStringFromBundle("dashboard.card.dataversemove.message.failure.summary"), + BundleUtil.getStringFromBundle("dashboard.card.dataversemove.message.failure.details", arguments))); + } + } + + public String getDataverseCount() { + long count = em.createQuery("SELECT count(dv) FROM Dataverse dv", Long.class).getSingleResult(); + return NumberFormat.getInstance().format(count); + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MoveDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MoveDatasetCommand.java index bee5dc648b9..9cf1dbc61de 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MoveDatasetCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MoveDatasetCommand.java @@ -72,13 +72,13 @@ public void executeImpl(CommandContext ctxt) throws CommandException { // validate the move makes sense if (moved.getOwner().equals(destination)) { - throw new IllegalCommandException(BundleUtil.getStringFromBundle("dashboard.card.datamove.dataset.command.error.targetDataverseSameAsOriginalDataverse"), this); + throw new IllegalCommandException(BundleUtil.getStringFromBundle("dashboard.card.datasetmove.dataset.command.error.targetDataverseSameAsOriginalDataverse"), this); } // if dataset is published make sure that its target is published if (moved.isReleased() && !destination.isReleased()){ - throw new IllegalCommandException(BundleUtil.getStringFromBundle("dashboard.card.datamove.dataset.command.error.targetDataverseUnpublishedDatasetPublished", Arrays.asList(destination.getDisplayName())), this); + throw new IllegalCommandException(BundleUtil.getStringFromBundle("dashboard.card.datasetmove.dataset.command.error.targetDataverseUnpublishedDatasetPublished", Arrays.asList(destination.getDisplayName())), this); } //if the datasets guestbook is not contained in the new dataverse then remove it @@ -130,10 +130,10 @@ public void executeImpl(CommandContext ctxt) throws CommandException { if (removeGuestbook || removeLinkDs) { StringBuilder errorString = new StringBuilder(); if (removeGuestbook) { - errorString.append(BundleUtil.getStringFromBundle("dashboard.card.datamove.dataset.command.error.unforced.datasetGuestbookNotInTargetDataverse")); + errorString.append(BundleUtil.getStringFromBundle("dashboard.card.datasetmove.dataset.command.error.unforced.datasetGuestbookNotInTargetDataverse")); } if (removeLinkDs) { - errorString.append(BundleUtil.getStringFromBundle("dashboard.card.datamove.dataset.command.error.unforced.linkedToTargetDataverseOrOneOfItsParents")); + errorString.append(BundleUtil.getStringFromBundle("dashboard.card.datasetmove.dataset.command.error.unforced.linkedToTargetDataverseOrOneOfItsParents")); } throw new UnforcedCommandException(errorString.toString(), this); } diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index c47356008ff..009d43ae8c1 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -542,6 +542,7 @@ dashboard.card.harvestingserver.sets={0, choice, 0#Sets|1#Set|2#Sets} dashboard.card.harvestingserver.btn.manage=Manage Server dashboard.card.metadataexport.header=Metadata Export dashboard.card.metadataexport.message=Dataset metadata export is only available through the {0} API. Learn more in the {0} {1}API Guide{2}. +dashboard.card.datamove=Data #harvestclients.xhtml harvestclients.title=Manage Harvesting Clients @@ -747,32 +748,51 @@ dashboard.list_users.toggleSuperuser.confirmationText.remove=Are you sure you wa dashboard.list_users.api.auth.invalid_apikey=The API key is invalid. dashboard.list_users.api.auth.not_superuser=Forbidden. You must be a superuser. -#dashboard-datamove.xhtml -dashboard.card.datamove=Data -dashboard.card.datamove.header=Dashboard - Move Data -dashboard.card.datamove.manage=Move Data -dashboard.card.datamove.message=Manage and curate your installation by moving datasets from one host dataverse to another. See also Managing Datasets and Dataverses in the Admin Guide. -dashboard.card.datamove.selectdataset.header=Dataset to Move -dashboard.card.datamove.newdataverse.header=New Host Dataverse -dashboard.card.datamove.dataset.label=Dataset -dashboard.card.datamove.dataverse.label=Dataverse -dashboard.card.datamove.confirm.dialog=Are you sure want to move this dataset? -dashboard.card.datamove.confirm.yes=Yes, Move Data -dashboard.card.datamove.message.success=The dataset "{0}" ({1}) has been successfully moved to {2}. -dashboard.card.datamove.message.failure.summary=Failed to moved dataset -dashboard.card.datamove.message.failure.details=The dataset "{0}" ({1}) could not be moved to {2}. {3}{4} -dashboard.card.datamove.dataverse.placeholder=Enter Dataverse Identifier... -dashboard.card.datamove.dataverse.menu.header=Dataverse Name (Affiliate), Identifier -dashboard.card.datamove.dataverse.menu.invalidMsg=No matches found -dashboard.card.datamove.dataset.placeholder=Enter Dataset Persistent ID, doi:... -dashboard.card.datamove.dataset.menu.header=Dataset Persistent ID, Title, Host Dataverse Identifier -dashboard.card.datamove.dataset.menu.invalidMsg=No matches found -dashboard.card.datamove.dataset.command.error.targetDataverseUnpublishedDatasetPublished=A published dataset may not be moved to an unpublished dataverse. You can retry the move after publishing {0}. -dashboard.card.datamove.dataset.command.error.targetDataverseSameAsOriginalDataverse=This dataset is already in this dataverse. -dashboard.card.datamove.dataset.command.error.unforced.datasetGuestbookNotInTargetDataverse=The guestbook would be removed from this dataset if you moved it because the guestbook is not in the new host dataverse. -dashboard.card.datamove.dataset.command.error.unforced.linkedToTargetDataverseOrOneOfItsParents=This dataset is linked to the new host dataverse or one of its parents. This move would remove the link to this dataset. -dashboard.card.datamove.dataset.command.error.unforced.suggestForce=Forcing this move is currently only available via API. Please see "Move a Dataset" under Managing Datasets and Dataverses in the Admin Guide for details. -dashboard.card.datamove.dataset.command.error.indexingProblem=Dataset could not be moved. Indexing failed. +#dashboard-datasetmove.xhtml +dashboard.card.datasetmove.header=Dashboard - Move Data +dashboard.card.datasetmove.manage=Move Dataset +dashboard.card.datasetmove.message=Manage and curate your installation by moving datasets from one host dataverse to another. See also Managing Datasets and Dataverses in the Admin Guide. +dashboard.card.datasetmove.selectdataset.header=Dataset to Move +dashboard.card.datasetmove.newdataverse.header=New Host Dataverse +dashboard.card.datasetmove.dataset.label=Dataset +dashboard.card.datasetmove.dataverse.label=Dataverse +dashboard.card.datasetmove.confirm.dialog=Are you sure want to move this dataset? +dashboard.card.datasetmove.confirm.yes=Yes, Move Dataset +dashboard.card.datsetamove.message.success=The dataset "{0}" ({1}) has been successfully moved to {2}. +dashboard.card.datasetmove.message.failure.summary=Failed to moved dataset +dashboard.card.datasetmove.message.failure.details=The dataset "{0}" ({1}) could not be moved to {2}. {3}{4} +dashboard.card.datasetmove.dataverse.placeholder=Enter Dataverse Identifier... +dashboard.card.datasetmove.dataverse.menu.header=Dataverse Name (Affiliate), Identifier +dashboard.card.datasetmove.dataverse.menu.invalidMsg=No matches found +dashboard.card.datasetmove.dataset.placeholder=Enter Dataset Persistent ID, doi:... +dashboard.card.datasetmove.dataset.menu.header=Dataset Persistent ID, Title, Host Dataverse Identifier +dashboard.card.datasetmove.dataset.menu.invalidMsg=No matches found +dashboard.card.datasetmove.dataset.command.error.targetDataverseUnpublishedDatasetPublished=A published dataset may not be moved to an unpublished dataverse. You can retry the move after publishing {0}. +dashboard.card.datasetmove.dataset.command.error.targetDataverseSameAsOriginalDataverse=This dataset is already in this dataverse. +dashboard.card.datasetmove.dataset.command.error.unforced.datasetGuestbookNotInTargetDataverse=The guestbook would be removed from this dataset if you moved it because the guestbook is not in the new host dataverse. +dashboard.card.datasetmove.dataset.command.error.unforced.linkedToTargetDataverseOrOneOfItsParents=This dataset is linked to the new host dataverse or one of its parents. This move would remove the link to this dataset. +dashboard.card.datasetmove.dataset.command.error.unforced.suggestForce=Forcing this move is currently only available via API. Please see "Move a Dataset" under Managing Datasets and Dataverses in the Admin Guide for details. + +#dashboard-dataversemove.xhtml +dashboard.card.dataversemove.header=Dashboard - Move Data +dashboard.card.dataversemove.manage=Move Collection +dashboard.card.dataversemove.message.summary=Move Dataverse Collection +dashboard.card.dataversemove.message.detail=Manage and curate your installation by moving a dataverse collection from one host dataverse collection to another. See also Managing Datasets and Dataverses in the Admin Guide. +dashboard.card.dataversemove.selectdataverse.header=Dataverse to Move +dashboard.card.dataversemove.newdataverse.header=New Host Dataverse +dashboard.card.dataversemove.dataverse.label=Dataverse +dashboard.card.dataversemove.confirm.dialog=Are you sure want to move this dataverse? +dashboard.card.dataversemove.confirm.yes=Yes, Move Dataverse +dashboard.card.dataversemove.message.success=The dataverse "{0}" has been successfully moved to {1}. +dashboard.card.dataversemove.message.failure.summary=Failed to moved dataverse +dashboard.card.dataversemove.message.failure.details=The dataverse "{0}" could not be moved to {1}. {2} +dashboard.card.dataversemove.dataverse.placeholder=Enter Dataverse Identifier... +dashboard.card.dataversemove.dataverse.menu.header=Dataverse Name (Affiliate), Identifier +dashboard.card.dataversemove.dataverse.menu.invalidMsg=No matches found +dashboard.card.dataversemove.dataset.command.error.targetDataverseUnpublishedDatasetPublished=A published dataset may not be moved to an unpublished dataverse. You can retry the move after publishing {0}. +dashboard.card.dataversemove.dataset.command.error.targetDataverseSameAsOriginalDataverse=This dataset is already in this dataverse. +dashboard.card.dataversemove.dataset.command.error.unforced.datasetGuestbookNotInTargetDataverse=The guestbook would be removed from this dataset if you moved it because the guestbook is not in the new host dataverse. +dashboard.card.dataversemove.dataset.command.error.unforced.linkedToTargetDataverseOrOneOfItsParents=This dataset is linked to the new host dataverse or one of its parents. This move would remove the link to this dataset. #MailServiceBean.java notification.email.create.dataverse.subject={0}: Your dataverse has been created diff --git a/src/main/webapp/dashboard-datamove.xhtml b/src/main/webapp/dashboard-datasetmove.xhtml similarity index 86% rename from src/main/webapp/dashboard-datamove.xhtml rename to src/main/webapp/dashboard-datasetmove.xhtml index 7f8365c9be3..96d9d401412 100644 --- a/src/main/webapp/dashboard-datamove.xhtml +++ b/src/main/webapp/dashboard-datasetmove.xhtml @@ -13,40 +13,40 @@ - + - + - +
-
#{bundle['dashboard.card.datamove.selectdataset.header']}
+
#{bundle['dashboard.card.datasetmove.selectdataset.header']}
@@ -70,26 +70,26 @@
-
#{bundle['dashboard.card.datamove.newdataverse.header']}
+
#{bundle['dashboard.card.datasetmove.newdataverse.header']}
@@ -127,12 +127,12 @@
- -

#{bundle['dashboard.card.datamove.confirm.dialog']}

+ +

#{bundle['dashboard.card.datasetmove.confirm.dialog']}

- +
+ + + + + + + diff --git a/src/main/webapp/dashboard.xhtml b/src/main/webapp/dashboard.xhtml index 5a72b52937b..f5b0536509d 100644 --- a/src/main/webapp/dashboard.xhtml +++ b/src/main/webapp/dashboard.xhtml @@ -129,18 +129,23 @@

#{bundle['dashboard.card.datamove']}

- -

#{bundle['dataverses']}

+ +

#{bundle['datasets']}

- -

#{bundle['datasets']}

+ +

#{bundle['dataverses']}

-
+
+

+ #{bundle['dashboard.card.datasetmove.manage']} +

+
+

- #{bundle['dashboard.card.datamove.manage']} + #{bundle['dashboard.card.dataversemove.manage']}

From 2c847be97bf6f25a187857585f3184a6451be3fd Mon Sep 17 00:00:00 2001 From: stevenferey Date: Wed, 4 Dec 2024 15:42:04 +0100 Subject: [PATCH 023/137] remove unused import --- .../iq/dataverse/dashboard/DashboardDataversemovePage.java | 7 ------- .../dataverse/engine/command/impl/MoveDatasetCommand.java | 2 -- .../engine/command/impl/MoveDataverseCommand.java | 3 --- 3 files changed, 12 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/dashboard/DashboardDataversemovePage.java b/src/main/java/edu/harvard/iq/dataverse/dashboard/DashboardDataversemovePage.java index df821dc953a..152d38c25e7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dashboard/DashboardDataversemovePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/dashboard/DashboardDataversemovePage.java @@ -1,7 +1,5 @@ package edu.harvard.iq.dataverse.dashboard; -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetServiceBean; import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.DataverseServiceBean; import edu.harvard.iq.dataverse.DataverseSession; @@ -11,8 +9,6 @@ import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; -import edu.harvard.iq.dataverse.engine.command.exception.UnforcedCommandException; -import edu.harvard.iq.dataverse.engine.command.impl.MoveDatasetCommand; import edu.harvard.iq.dataverse.engine.command.impl.MoveDataverseCommand; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.JsfHelper; @@ -43,9 +39,6 @@ public class DashboardDataversemovePage implements java.io.Serializable { PermissionsWrapper permissionsWrapper; @EJB EjbDataverseEngine commandEngine; - - //@EJB - //DatasetServiceBean datasetService; @EJB DataverseServiceBean dataverseService; @Inject diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MoveDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MoveDatasetCommand.java index 9cf1dbc61de..b6a2793b8e0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MoveDatasetCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MoveDatasetCommand.java @@ -10,7 +10,6 @@ import edu.harvard.iq.dataverse.DatasetLock; import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.Guestbook; -import edu.harvard.iq.dataverse.authorization.DataverseRole; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.engine.command.AbstractVoidCommand; @@ -27,7 +26,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.logging.Level; import java.util.logging.Logger; /** diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MoveDataverseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MoveDataverseCommand.java index ea38f5a7af7..c8b59b1818a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MoveDataverseCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MoveDataverseCommand.java @@ -10,7 +10,6 @@ import edu.harvard.iq.dataverse.Template; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; -import edu.harvard.iq.dataverse.batch.util.LoggingUtil; import edu.harvard.iq.dataverse.engine.command.AbstractVoidCommand; import edu.harvard.iq.dataverse.engine.command.CommandContext; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; @@ -20,14 +19,12 @@ import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; import edu.harvard.iq.dataverse.engine.command.exception.PermissionException; import edu.harvard.iq.dataverse.util.BundleUtil; -import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.logging.Logger; -import org.apache.solr.client.solrj.SolrServerException; /** * A command to move a {@link Dataverse} between two {@link Dataverse}s. From b94d53fe1a5fe7a861fb0b5c4b180d76625b40dd Mon Sep 17 00:00:00 2001 From: Pierre Le Corre <146710476+plecor@users.noreply.github.com> Date: Wed, 4 Dec 2024 17:32:54 +0100 Subject: [PATCH 024/137] Improve release note --- doc/release-notes/8739-publisher-during-harvesting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/8739-publisher-during-harvesting.md b/doc/release-notes/8739-publisher-during-harvesting.md index 05dcc53ac7d..3e1555396c0 100644 --- a/doc/release-notes/8739-publisher-during-harvesting.md +++ b/doc/release-notes/8739-publisher-during-harvesting.md @@ -1 +1 @@ -The publisher value of harvested datasets is now attributed to the dataset's distributor instead of its producer. This change affects all newly harvested datasets. For more information, see [the guides](https://dataverse-guide--9013.org.readthedocs.build/en/9013/admin/harvestclients.html#harvesting-client-changelog), #8739, and #9013. \ No newline at end of file +The publisher value of harvested datasets is now attributed to the dataset's distributor instead of its producer. This improves the citation associated with these datasets, but the change only affects newly harvested datasets. All datasets should be re-harvested if you wish to pick up this change on already harvested datasets. For more information, see [the guides](https://dataverse-guide--9013.org.readthedocs.build/en/9013/admin/harvestclients.html#harvesting-client-changelog), #8739, and #9013. \ No newline at end of file From b23d82b00d9797779d6771c6061cf1c1deec1cb9 Mon Sep 17 00:00:00 2001 From: Pierre Le Corre <146710476+plecor@users.noreply.github.com> Date: Thu, 5 Dec 2024 09:53:25 +0100 Subject: [PATCH 025/137] Rename migration file --- ...6.4.0.1__8739-publisher-during-harvesting.sql => V6.5.0.1.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V6.4.0.1__8739-publisher-during-harvesting.sql => V6.5.0.1.sql} (100%) diff --git a/src/main/resources/db/migration/V6.4.0.1__8739-publisher-during-harvesting.sql b/src/main/resources/db/migration/V6.5.0.1.sql similarity index 100% rename from src/main/resources/db/migration/V6.4.0.1__8739-publisher-during-harvesting.sql rename to src/main/resources/db/migration/V6.5.0.1.sql From c8499ba9553ac46cf3adc56d1b9e56f0c781d30f Mon Sep 17 00:00:00 2001 From: Pierre Le Corre <146710476+plecor@users.noreply.github.com> Date: Mon, 9 Dec 2024 16:52:09 +0100 Subject: [PATCH 026/137] Applies sql change to upsert script --- .../db/migration/afterMigrate__1-7256-upsert-referenceData.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/db/migration/afterMigrate__1-7256-upsert-referenceData.sql b/src/main/resources/db/migration/afterMigrate__1-7256-upsert-referenceData.sql index 07e9b2c6266..f63fedba02f 100644 --- a/src/main/resources/db/migration/afterMigrate__1-7256-upsert-referenceData.sql +++ b/src/main/resources/db/migration/afterMigrate__1-7256-upsert-referenceData.sql @@ -31,7 +31,7 @@ INSERT INTO foreignmetadatafieldmapping (id, foreignfieldxpath, datasetfieldname (15, 'affiliation', 'authorAffiliation', TRUE, 3, 1 ), (16, ':contributor', 'contributorName', FALSE, NULL, 1 ), (17, 'type', 'contributorType', TRUE, 16, 1 ), - (18, ':publisher', 'producerName', FALSE, NULL, 1 ), + (18, ':publisher', 'distributorName', FALSE, NULL, 1 ), (19, ':language', 'language', FALSE, NULL, 1 ) ON CONFLICT DO NOTHING; From 5460bdd27c180aeb0542b5fb66d060b6816a68ee Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 12 Dec 2024 15:27:10 -0500 Subject: [PATCH 027/137] account for cvoc multiples --- src/main/java/edu/harvard/iq/dataverse/api/Index.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Index.java b/src/main/java/edu/harvard/iq/dataverse/api/Index.java index c30a77acb58..0b3e3ac52d1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Index.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Index.java @@ -44,6 +44,7 @@ import java.lang.reflect.Field; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; @@ -451,11 +452,11 @@ public Response clearOrphans(@QueryParam("sync") String sync) { public String getSolrSchema() { StringBuilder sb = new StringBuilder(); - - for (DatasetFieldType datasetField : datasetFieldService.findAllOrderedByName()) { + Map cvocTermUriMap = datasetFieldSvc.getCVocConf(true); + for (DatasetFieldType datasetFieldType : datasetFieldService.findAllOrderedByName()) { //ToDo - getSolrField() creates/returns a new object - just get it once and re-use - String nameSearchable = datasetField.getSolrField().getNameSearchable(); - SolrField.SolrType solrType = datasetField.getSolrField().getSolrType(); + String nameSearchable = datasetFieldType.getSolrField().getNameSearchable(); + SolrField.SolrType solrType = datasetFieldType.getSolrField().getSolrType(); String type = solrType.getType(); if (solrType.equals(SolrField.SolrType.EMAIL)) { /** @@ -474,7 +475,7 @@ public String getSolrSchema() { */ logger.info("email type detected (" + nameSearchable + ") See also https://github.com/IQSS/dataverse/issues/759"); } - String multivalued = datasetField.getSolrField().isAllowedToBeMultivalued().toString(); + String multivalued = Boolean.toString(datasetFieldType.getSolrField().isAllowedToBeMultivalued()|| cvocTermUriMap.containsKey(datasetFieldType.getId())); // sb.append(" \n"); } From f85c26c31e629a6ce9321f445b87d8234ac57b44 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 12 Dec 2024 16:50:24 -0500 Subject: [PATCH 028/137] release note, docs --- doc/release-notes/11095-fix-extcvoc-indexing.md | 7 +++++++ doc/sphinx-guides/source/admin/metadatacustomization.rst | 6 ++++-- doc/sphinx-guides/source/installation/config.rst | 3 +++ 3 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 doc/release-notes/11095-fix-extcvoc-indexing.md diff --git a/doc/release-notes/11095-fix-extcvoc-indexing.md b/doc/release-notes/11095-fix-extcvoc-indexing.md new file mode 100644 index 00000000000..1459b88c122 --- /dev/null +++ b/doc/release-notes/11095-fix-extcvoc-indexing.md @@ -0,0 +1,7 @@ +Some External Controlled Vocabulary scripts/configurations, when used on a metadata field that is single valued could result +in indexing failure for the dataset (e.g. when the the script tried to index both the identifier and name of the identified entity for indexing). +Dataverse has been updated to correctly indicate the need for a multi-valued solr field in these cases in the call to /api/admin/index/solr/schema. +Configuring the Solr schema and the update-fields.sh script as usually recommended when using custom metadata blocks will resolve the issue. + +The overall release notes should include a solr update (which hopefully is required by an update to 9.7.0 anyway) and our standard instructions +should change to recommending use of the udpate-fields.sh script when using custom metadatablocks *and/or external vocabulary scripts*. diff --git a/doc/sphinx-guides/source/admin/metadatacustomization.rst b/doc/sphinx-guides/source/admin/metadatacustomization.rst index e5326efebef..eee83260804 100644 --- a/doc/sphinx-guides/source/admin/metadatacustomization.rst +++ b/doc/sphinx-guides/source/admin/metadatacustomization.rst @@ -559,8 +559,7 @@ Using External Vocabulary Services The Dataverse software has a mechanism to associate specific fields defined in metadata blocks with a vocabulary(ies) managed by external services. The mechanism relies on trusted third-party Javascripts. The mapping from field type to external vocabulary(ies) is managed via the :ref:`:CVocConf <:CVocConf>` setting. -*This functionality is considered 'experimental'. It may require significant effort to configure and is likely to evolve in subsequent Dataverse software releases.* - +*This functionality may require significant effort to configure and is likely to evolve in subsequent Dataverse software releases.* The effect of configuring this mechanism is similar to that of defining a field in a metadata block with 'allowControlledVocabulary=true': @@ -585,6 +584,9 @@ Configuration involves specifying which fields are to be mapped, to which Solr f These are all defined in the :ref:`:CVocConf <:CVocConf>` setting as a JSON array. Details about the required elements as well as example JSON arrays are available at https://github.com/gdcc/dataverse-external-vocab-support, along with an example metadata block that can be used for testing. The scripts required can be hosted locally or retrieved dynamically from https://gdcc.github.io/ (similar to how dataverse-previewers work). +Since external vocabulary scripts can change how fields are indexed (storing an identifier and name and/or values in different languages), +updating the solr schema as described in :ref:`update-solr-schema` should be done after adding new scripts to your configuration. + Please note that in addition to the :ref:`:CVocConf` described above, an alternative is the :ref:`:ControlledVocabularyCustomJavaScript` setting. Protecting MetadataBlocks diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 30a36da9499..a310e9f96f8 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -4653,6 +4653,9 @@ The commands below should give you an idea of how to load the configuration, but ``curl -X PUT --upload-file cvoc-conf.json http://localhost:8080/api/admin/settings/:CVocConf`` +Since external vocabulary scripts can change how fields are indexed (storing an identifier and name and/or values in different languages), +updating the solr schema as described in :ref:`update-solr-schema` should be done after adding new scripts to your configuration. + .. _:ControlledVocabularyCustomJavaScript: :ControlledVocabularyCustomJavaScript From 243ba50a45c9be4fdd8e66cef1697cda95dc44af Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Mon, 16 Dec 2024 08:32:16 -0500 Subject: [PATCH 029/137] rename flyway script --- src/main/resources/db/migration/{V6.4.0.4.sql => V6.5.0.1.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V6.4.0.4.sql => V6.5.0.1.sql} (100%) diff --git a/src/main/resources/db/migration/V6.4.0.4.sql b/src/main/resources/db/migration/V6.5.0.1.sql similarity index 100% rename from src/main/resources/db/migration/V6.4.0.4.sql rename to src/main/resources/db/migration/V6.5.0.1.sql From d796959c8cd1ce0ef05bc8474a5da109c6219d02 Mon Sep 17 00:00:00 2001 From: GPortas Date: Mon, 16 Dec 2024 17:51:57 +0100 Subject: [PATCH 030/137] Stash: update featureItems endpoint PoC WIP --- .../harvard/iq/dataverse/api/Dataverses.java | 56 ++++++++++++++++++- src/main/java/propertyFiles/Bundle.properties | 1 + .../iq/dataverse/api/DataversesIT.java | 13 +++++ .../edu/harvard/iq/dataverse/api/UtilIT.java | 10 ++++ 4 files changed, 77 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index f864a5a9d1c..e7ca7033210 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -24,6 +24,7 @@ import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.ConstraintViolationUtil; +import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.StringUtil; import static edu.harvard.iq.dataverse.util.StringUtil.nonEmpty; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.*; @@ -33,7 +34,9 @@ import edu.harvard.iq.dataverse.util.json.JsonPrinter; import edu.harvard.iq.dataverse.util.json.JsonUtil; -import java.io.StringReader; +import java.io.*; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; import java.util.*; import java.util.logging.Level; import java.util.logging.Logger; @@ -59,8 +62,6 @@ import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.Response.Status; -import java.io.IOException; -import java.io.OutputStream; import java.text.MessageFormat; import java.text.SimpleDateFormat; import java.util.stream.Collectors; @@ -68,6 +69,9 @@ import jakarta.ws.rs.WebApplicationException; import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.StreamingOutput; +import org.glassfish.jersey.media.multipart.FormDataContentDisposition; +import org.glassfish.jersey.media.multipart.FormDataParam; + import javax.xml.stream.XMLStreamException; /** @@ -1729,4 +1733,50 @@ public Response getUserPermissionsOnDataverse(@Context ContainerRequestContext c jsonObjectBuilder.add("canDeleteDataverse", permissionService.userOn(requestUser, dataverse).has(Permission.DeleteDataverse)); return ok(jsonObjectBuilder); } + + @PUT + @AuthRequired + @Consumes(MediaType.MULTIPART_FORM_DATA) + @Path("{identifier}/featuredItems") + public Response updateFeaturedItems(@Context ContainerRequestContext crc, + @PathParam("identifier") String dvIdtf, + @FormDataParam("title") String title, + @FormDataParam("content") String content, + @FormDataParam("file") InputStream fileInputStream, + @FormDataParam("file") FormDataContentDisposition contentDispositionHeader) { + Dataverse dataverse; + try { + dataverse = findDataverseOrDie(dvIdtf); + } catch (WrappedResponse wr) { + return wr.getResponse(); + } + try { + String fileName = contentDispositionHeader.getFileName(); + File uploadedFile = new File(createTempDir(dataverse), fileName); + if (!uploadedFile.exists()) { + uploadedFile.createNewFile(); + } + File file = FileUtil.inputStreamToFile(fileInputStream); + if (file.length() > systemConfig.getUploadLogoSizeLimit()) { + return error(Response.Status.BAD_REQUEST, "File is larger than maximum size: " + systemConfig.getUploadLogoSizeLimit() + "."); + } + Files.copy(fileInputStream, uploadedFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + throw new RuntimeException(e); + } + return ok(""); + } + + private File createTempDir(Dataverse editDv) { + try { + // Create the temporary space if not yet existing (will silently ignore preexisting) + // Note that the docroot directory is checked within ConfigCheckService for presence and write access. + java.nio.file.Path tempRoot = java.nio.file.Path.of(JvmSettings.DOCROOT_DIRECTORY.lookup(), "featuredItems"); + Files.createDirectories(tempRoot); + + return Files.createTempDirectory(tempRoot, editDv.getId().toString()).toFile(); + } catch (IOException e) { + throw new RuntimeException("Error creating temp directory", e); // improve error handling + } + } } diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index c47356008ff..193e4d2263f 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -2790,6 +2790,7 @@ dataverses.api.create.dataset.error.mustIncludeAuthorName=Please provide author dataverses.api.validate.json.succeeded=The Dataset JSON provided is valid for this Dataverse Collection. dataverses.api.validate.json.failed=The Dataset JSON provided failed validation with the following error: dataverses.api.validate.json.exception=Validation failed with following exception: +dataverses.api.update.featured.items.error.onlyImageFilesAllowed=Invalid file type. Only image files are allowed. #Access.java access.api.allowRequests.failure.noDataset=Could not find Dataset with id: {0} diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 76bb515beb2..53f9b1bf06d 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -1559,4 +1559,17 @@ public void testGetUserPermissionsOnDataverse() { Response getUserPermissionsOnDataverseInvalidIdResponse = UtilIT.getUserPermissionsOnDataverse("testInvalidAlias", apiToken); getUserPermissionsOnDataverseInvalidIdResponse.then().assertThat().statusCode(NOT_FOUND.getStatusCode()); } + + @Test + public void testUpdateFeaturedItems() { + Response createUserResponse = UtilIT.createRandomUser(); + String apiToken = UtilIT.getApiTokenFromResponse(createUserResponse); + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode()); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + String pathToTestFile = "src/test/resources/images/coffeeshop.png"; + Response updateFeatureItemsResponse = UtilIT.updateFeaturedItems(dataverseAlias, apiToken, "test", "test", pathToTestFile); + updateFeatureItemsResponse.prettyPrint(); + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 1930610532a..c7886124191 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -4305,4 +4305,14 @@ static Response deleteDatasetTypes(long doomed, String apiToken) { .delete("/api/datasets/datasetTypes/" + doomed); } + static Response updateFeaturedItems(String dataverseAlias, String apiToken, String title, String content, String pathToFile) { + return given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .contentType(ContentType.MULTIPART) + .multiPart("title", title) + .multiPart("content", content) + .multiPart("file", new File(pathToFile)) + .when() + .put("/api/dataverses/" + dataverseAlias + "/featuredItems"); + } } From c1a32765abd5522a9bfc645328ea675dd6f1f83a Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Mon, 16 Dec 2024 14:06:36 -0500 Subject: [PATCH 031/137] rename sql file --- src/main/resources/db/migration/{V6.5.0.1.sql => V6.5.0.2.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V6.5.0.1.sql => V6.5.0.2.sql} (100%) diff --git a/src/main/resources/db/migration/V6.5.0.1.sql b/src/main/resources/db/migration/V6.5.0.2.sql similarity index 100% rename from src/main/resources/db/migration/V6.5.0.1.sql rename to src/main/resources/db/migration/V6.5.0.2.sql From 45517794500fe34e17015521344e073f59df351d Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 16 Dec 2024 14:32:53 -0500 Subject: [PATCH 032/137] Update auth.rst suggested edits --- doc/sphinx-guides/source/api/auth.rst | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/doc/sphinx-guides/source/api/auth.rst b/doc/sphinx-guides/source/api/auth.rst index 210c1bcd184..5acf0fb0ff7 100644 --- a/doc/sphinx-guides/source/api/auth.rst +++ b/doc/sphinx-guides/source/api/auth.rst @@ -87,22 +87,20 @@ To register a new user who has authenticated via an OIDC provider, the following curl -H "Authorization: Bearer $TOKEN" -X POST http://localhost:8080/api/users/register --data '{"termsAccepted":true}' -If the feature flag ``api-bearer-auth-handle-tos-acceptance-in-idp``` is disabled, it is essential to send a JSON that includes the property ``termsAccepted``` set to true, indicating that you accept the Terms of Use of the installation. Otherwise, you will not be able to create an account. However, if the feature flag is enabled, Terms of Service acceptance is handled by the identity provider, and it is no longer necessary to include the ``termsAccepted``` parameter in the JSON. - -In this JSON, we can also include the fields ``position`` or ``affiliation``, in the same way as when we register a user through the Dataverse UI. These fields are optional, and if not provided, they will be persisted as empty in Dataverse. - -There is another flag called ``api-bearer-auth-provide-missing-claims`` that can be enabled to allow sending missing user claims in the registration JSON. This is useful when the identity provider does not supply the necessary claims. However, this flag will only be considered if the ``api-bearer-auth`` feature flag is enabled. If the latter is not enabled, the ``api-bearer-auth-provide-missing-claims`` flag will be ignored. - -With the ``api-bearer-auth-provide-missing-claims`` feature flag enabled, you can include the following properties in the request JSON: +By default, the Bearer token is expected to include the following claims that will be used to create the user account: - ``username`` - ``firstName`` - ``lastName`` - ``emailAddress`` -If properties are provided in the JSON, but corresponding claims already exist in the identity provider, an error will be thrown, outlining the conflicting properties. +The one parameter required by default is ``termsAccepted``` which must be set to true, indicating that the user has seen and accepted the Terms of Use of the Installation. + +If the feature flag ``api-bearer-auth-handle-tos-acceptance-in-idp``` is enabled (along with the ``api-bearer-auth`` feature flag), Dataverse assumes that the Terms of Service acceptance was handled by the identity provider, e.g. in the OIDC ``consent``` dialog, and the `termsAccepted``` parameter is not needed. + +There is another flag called ``api-bearer-auth-provide-missing-claims`` that can be enabled (along with the ``api-bearer-auth`` feature flag) to allow sending missing user claims in the registration JSON. This is useful when the identity provider does not supply the necessary claims listed above. If properties are provided in the JSON, but corresponding claims already exist in the identity provider, an error will be thrown, outlining the conflicting properties. Note that supplying missing claims is configured via a separate feature flag because using it may introduce user impersonation issues, for example if the identity provider does not provide an email field and the user submits an email address they do not own. -This functionality is included under a feature flag because using it may introduce user impersonation issues, for example if the identity provider does not provide an email field and the user submits an email address they do not own. +In all cases, the submitted JSON can optionally include the fields ``position`` or ``affiliation``, which will be added to the users's Dataverse account profile. These fields are optional, and if not provided, they will be persisted as empty in Dataverse. Signed URLs ----------- From 968dfdaf9372065f885efb4a94d81b40ce6ec06f Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 19 Dec 2024 12:41:00 -0500 Subject: [PATCH 033/137] Apply suggestions from code review Co-authored-by: Philip Durbin --- doc/release-notes/11095-fix-extcvoc-indexing.md | 10 +++++----- .../source/admin/metadatacustomization.rst | 2 +- doc/sphinx-guides/source/installation/config.rst | 2 +- src/main/java/edu/harvard/iq/dataverse/api/Index.java | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/release-notes/11095-fix-extcvoc-indexing.md b/doc/release-notes/11095-fix-extcvoc-indexing.md index 1459b88c122..f4931d81263 100644 --- a/doc/release-notes/11095-fix-extcvoc-indexing.md +++ b/doc/release-notes/11095-fix-extcvoc-indexing.md @@ -1,7 +1,7 @@ -Some External Controlled Vocabulary scripts/configurations, when used on a metadata field that is single valued could result -in indexing failure for the dataset (e.g. when the the script tried to index both the identifier and name of the identified entity for indexing). -Dataverse has been updated to correctly indicate the need for a multi-valued solr field in these cases in the call to /api/admin/index/solr/schema. +Some External Controlled Vocabulary scripts/configurations, when used on a metadata field that is single-valued could result +in indexing failure for the dataset (e.g. when the script tried to index both the identifier and name of the identified entity for indexing). +Dataverse has been updated to correctly indicate the need for a multi-valued Solr field in these cases in the call to /api/admin/index/solr/schema. Configuring the Solr schema and the update-fields.sh script as usually recommended when using custom metadata blocks will resolve the issue. -The overall release notes should include a solr update (which hopefully is required by an update to 9.7.0 anyway) and our standard instructions -should change to recommending use of the udpate-fields.sh script when using custom metadatablocks *and/or external vocabulary scripts*. +The overall release notes should include a Solr update (which hopefully is required by an update to 9.7.0 anyway) and our standard instructions +should change to recommending use of the update-fields.sh script when using custom metadatablocks *and/or external vocabulary scripts*. diff --git a/doc/sphinx-guides/source/admin/metadatacustomization.rst b/doc/sphinx-guides/source/admin/metadatacustomization.rst index eee83260804..2a104354af9 100644 --- a/doc/sphinx-guides/source/admin/metadatacustomization.rst +++ b/doc/sphinx-guides/source/admin/metadatacustomization.rst @@ -585,7 +585,7 @@ These are all defined in the :ref:`:CVocConf <:CVocConf>` setting as a JSON arra The scripts required can be hosted locally or retrieved dynamically from https://gdcc.github.io/ (similar to how dataverse-previewers work). Since external vocabulary scripts can change how fields are indexed (storing an identifier and name and/or values in different languages), -updating the solr schema as described in :ref:`update-solr-schema` should be done after adding new scripts to your configuration. +updating the Solr schema as described in :ref:`update-solr-schema` should be done after adding new scripts to your configuration. Please note that in addition to the :ref:`:CVocConf` described above, an alternative is the :ref:`:ControlledVocabularyCustomJavaScript` setting. diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index a310e9f96f8..a653a100c89 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -4654,7 +4654,7 @@ The commands below should give you an idea of how to load the configuration, but ``curl -X PUT --upload-file cvoc-conf.json http://localhost:8080/api/admin/settings/:CVocConf`` Since external vocabulary scripts can change how fields are indexed (storing an identifier and name and/or values in different languages), -updating the solr schema as described in :ref:`update-solr-schema` should be done after adding new scripts to your configuration. +updating the Solr schema as described in :ref:`update-solr-schema` should be done after adding new scripts to your configuration. .. _:ControlledVocabularyCustomJavaScript: diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Index.java b/src/main/java/edu/harvard/iq/dataverse/api/Index.java index 0b3e3ac52d1..bc9a8ae692b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Index.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Index.java @@ -475,7 +475,7 @@ public String getSolrSchema() { */ logger.info("email type detected (" + nameSearchable + ") See also https://github.com/IQSS/dataverse/issues/759"); } - String multivalued = Boolean.toString(datasetFieldType.getSolrField().isAllowedToBeMultivalued()|| cvocTermUriMap.containsKey(datasetFieldType.getId())); + String multivalued = Boolean.toString(datasetFieldType.getSolrField().isAllowedToBeMultivalued() || cvocTermUriMap.containsKey(datasetFieldType.getId())); // sb.append(" \n"); } From 826e9bf0c1e87991d7b42313e89e4d0c51d50cc2 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 19 Dec 2024 15:33:00 -0500 Subject: [PATCH 034/137] add ROR to authorIdentifierScheme #11075 --- scripts/api/data/metadatablocks/citation.tsv | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/api/data/metadatablocks/citation.tsv b/scripts/api/data/metadatablocks/citation.tsv index abc09465603..a98f16c64c7 100644 --- a/scripts/api/data/metadatablocks/citation.tsv +++ b/scripts/api/data/metadatablocks/citation.tsv @@ -140,6 +140,7 @@ authorIdentifierScheme DAI 5 authorIdentifierScheme ResearcherID 6 authorIdentifierScheme ScopusID 7 + authorIdentifierScheme ROR 8 language 'Are'are alu 0 alu language 'Auhelawa kud 1 kud language A'ou aou 2 aou @@ -8061,4 +8062,4 @@ publicationRelationType IsSupplementTo RT3 3 publicationRelationType IsSupplementedBy RT4 4 publicationRelationType IsReferencedBy RT5 5 - publicationRelationType References RT6 6 \ No newline at end of file + publicationRelationType References RT6 6 From 29902a39a892fcbdbdba9a406c6b96a9c426aeec Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 19 Dec 2024 15:42:31 -0500 Subject: [PATCH 035/137] add release note #11075 --- doc/release-notes/11075-ror.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 doc/release-notes/11075-ror.md diff --git a/doc/release-notes/11075-ror.md b/doc/release-notes/11075-ror.md new file mode 100644 index 00000000000..711488ed8cb --- /dev/null +++ b/doc/release-notes/11075-ror.md @@ -0,0 +1,15 @@ +### ROR (Research Organization Registry) as Author Identifier Type + +ROR (Research Organization Registry) has been added as an Author Identifier Type (alongside ORCID, etc.). See #11075 and #11118. + +## Upgrade Instructions + +6\. Update metadata blocks + +These changes reflect incremental improvements made to the handling of core metadata fields. + +```shell +wget https://raw.githubusercontent.com/IQSS/dataverse/v6.4/scripts/api/data/metadatablocks/citation.tsv + +curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file citation.tsv +``` From 0d5665c25d98ff790588c36310756243ad02dec6 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 20 Dec 2024 10:54:51 -0500 Subject: [PATCH 036/137] add ROR to citation.properties #11075 --- src/main/java/propertyFiles/citation.properties | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/propertyFiles/citation.properties b/src/main/java/propertyFiles/citation.properties index 5899523da67..3e5e75ec94a 100644 --- a/src/main/java/propertyFiles/citation.properties +++ b/src/main/java/propertyFiles/citation.properties @@ -305,6 +305,7 @@ controlledvocabulary.authorIdentifierScheme.gnd=GND controlledvocabulary.authorIdentifierScheme.dai=DAI controlledvocabulary.authorIdentifierScheme.researcherid=ResearcherID controlledvocabulary.authorIdentifierScheme.scopusid=ScopusID +controlledvocabulary.authorIdentifierScheme.ror=ROR controlledvocabulary.language.'are'are='Are'are controlledvocabulary.language.'auhelawa='Auhelawa controlledvocabulary.language.a'ou=A'ou From 029c2477b7dcbd70d535914bd2525f43d0207c51 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 20 Dec 2024 11:58:49 -0500 Subject: [PATCH 037/137] update display order of Author Identifier (ORCID then ROR) #11075 --- scripts/api/data/metadatablocks/citation.tsv | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/api/data/metadatablocks/citation.tsv b/scripts/api/data/metadatablocks/citation.tsv index a98f16c64c7..dea23aa9a73 100644 --- a/scripts/api/data/metadatablocks/citation.tsv +++ b/scripts/api/data/metadatablocks/citation.tsv @@ -133,14 +133,14 @@ contributorType Work Package Leader 15 contributorType Other 16 authorIdentifierScheme ORCID 0 - authorIdentifierScheme ISNI 1 - authorIdentifierScheme LCNA 2 - authorIdentifierScheme VIAF 3 - authorIdentifierScheme GND 4 - authorIdentifierScheme DAI 5 - authorIdentifierScheme ResearcherID 6 - authorIdentifierScheme ScopusID 7 - authorIdentifierScheme ROR 8 + authorIdentifierScheme ROR 1 + authorIdentifierScheme ISNI 2 + authorIdentifierScheme LCNA 3 + authorIdentifierScheme VIAF 4 + authorIdentifierScheme GND 5 + authorIdentifierScheme DAI 6 + authorIdentifierScheme ResearcherID 7 + authorIdentifierScheme ScopusID 8 language 'Are'are alu 0 alu language 'Auhelawa kud 1 kud language A'ou aou 2 aou From 335cf7288fa84090dd8bda3c44df05a0606b23b7 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 20 Dec 2024 11:59:54 -0500 Subject: [PATCH 038/137] bump to next version: 6.6 #11075 --- doc/release-notes/11075-ror.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/11075-ror.md b/doc/release-notes/11075-ror.md index 711488ed8cb..0f2460b8df9 100644 --- a/doc/release-notes/11075-ror.md +++ b/doc/release-notes/11075-ror.md @@ -9,7 +9,7 @@ ROR (Research Organization Registry) has been added as an Author Identifier Type These changes reflect incremental improvements made to the handling of core metadata fields. ```shell -wget https://raw.githubusercontent.com/IQSS/dataverse/v6.4/scripts/api/data/metadatablocks/citation.tsv +wget https://raw.githubusercontent.com/IQSS/dataverse/v6.6/scripts/api/data/metadatablocks/citation.tsv curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file citation.tsv ``` From ddb6bc96a39ab2f758d1dce1e1ea234bee2f44d2 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 20 Dec 2024 12:12:37 -0500 Subject: [PATCH 039/137] put ROR after ORCID to match displayOrder #11075 --- src/main/java/propertyFiles/citation.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/propertyFiles/citation.properties b/src/main/java/propertyFiles/citation.properties index 3e5e75ec94a..9a1e6f280ec 100644 --- a/src/main/java/propertyFiles/citation.properties +++ b/src/main/java/propertyFiles/citation.properties @@ -298,6 +298,7 @@ controlledvocabulary.contributorType.supervisor=Supervisor controlledvocabulary.contributorType.work_package_leader=Work Package Leader controlledvocabulary.contributorType.other=Other controlledvocabulary.authorIdentifierScheme.orcid=ORCID +controlledvocabulary.authorIdentifierScheme.ror=ROR controlledvocabulary.authorIdentifierScheme.isni=ISNI controlledvocabulary.authorIdentifierScheme.lcna=LCNA controlledvocabulary.authorIdentifierScheme.viaf=VIAF @@ -305,7 +306,6 @@ controlledvocabulary.authorIdentifierScheme.gnd=GND controlledvocabulary.authorIdentifierScheme.dai=DAI controlledvocabulary.authorIdentifierScheme.researcherid=ResearcherID controlledvocabulary.authorIdentifierScheme.scopusid=ScopusID -controlledvocabulary.authorIdentifierScheme.ror=ROR controlledvocabulary.language.'are'are='Are'are controlledvocabulary.language.'auhelawa='Auhelawa controlledvocabulary.language.a'ou=A'ou From 6ae9c5571ca77bd60ccb6b744c3e4bcee448ce63 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 20 Dec 2024 12:16:30 -0500 Subject: [PATCH 040/137] explain ROR use case, link to https://ror.org #11075 --- doc/release-notes/11075-ror.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/11075-ror.md b/doc/release-notes/11075-ror.md index 0f2460b8df9..014ced2dd73 100644 --- a/doc/release-notes/11075-ror.md +++ b/doc/release-notes/11075-ror.md @@ -1,6 +1,6 @@ ### ROR (Research Organization Registry) as Author Identifier Type -ROR (Research Organization Registry) has been added as an Author Identifier Type (alongside ORCID, etc.). See #11075 and #11118. +ROR (Research Organization Registry) has been added as an Author Identifier Type (alongside ORCID, etc.) for when the author is an organization rather than a person. See , #11075, and #11118. ## Upgrade Instructions From a90b9b2185d21e2bdc9f1c2830afa700b6fa1f2a Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 19 Dec 2024 15:01:27 -0500 Subject: [PATCH 041/137] respond with 403 (not 401) when password good but no perms #10340 --- .../iq/dataverse/api/AbstractApiBean.java | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index 3c1074b75bb..8a88ff042ab 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -831,6 +831,18 @@ protected Response badRequest(String msg, Map fieldErrors) { .build(); } + /** + * In short, your password is fine but you don't have permission. + * + * "The 403 (Forbidden) status code indicates that the server understood the + * request but refuses to authorize it. A server that wishes to make public + * why the request has been forbidden can describe that reason in the + * response payload (if any). + * + * If authentication credentials were provided in the request, the server + * considers them insufficient to grant access." -- + * https://datatracker.ietf.org/doc/html/rfc7231#section-6.5.3 + */ protected Response forbidden( String msg ) { return error( Status.FORBIDDEN, msg ); } @@ -852,9 +864,17 @@ protected Response permissionError( PermissionException pe ) { } protected Response permissionError( String message ) { - return unauthorized( message ); + return forbidden( message ); } + /** + * In short, bad password. + * + * "The 401 (Unauthorized) status code indicates that the request has not + * been applied because it lacks valid authentication credentials for the + * target resource." -- + * https://datatracker.ietf.org/doc/html/rfc7235#section-3.1 + */ protected Response unauthorized( String message ) { return error( Status.UNAUTHORIZED, message ); } From bc7cb235ecf93b56786d210c9633c27dc3f3c2e8 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 19 Dec 2024 15:15:41 -0500 Subject: [PATCH 042/137] add release note #10340 --- doc/release-notes/10340-forbidden.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 doc/release-notes/10340-forbidden.md diff --git a/doc/release-notes/10340-forbidden.md b/doc/release-notes/10340-forbidden.md new file mode 100644 index 00000000000..0910646c7bd --- /dev/null +++ b/doc/release-notes/10340-forbidden.md @@ -0,0 +1,7 @@ +### API Now Returns 403 Forbidden for Permission Checks + +Dataverse was returning 401 Unauthorized when a permission check failed. This has been corrected to return 403 Forbidden in these cases. That is, the API token is known to be good (401 otherwise) but the user lacks permission (403 is now sent). See also #10340 and #11116. + +### Backward Incompatible Changes + +See "API Now Returns 403 Forbidden for Permission Checks" above. From caa00fb166c9197f154663dd52af54c78da88be1 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 20 Dec 2024 13:52:43 -0500 Subject: [PATCH 043/137] add test to exercise change from unauth to forbidden #10340 --- .../java/edu/harvard/iq/dataverse/api/RolesIT.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/RolesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/RolesIT.java index d15fda3a1a1..7e0a4714b1f 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/RolesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/RolesIT.java @@ -4,6 +4,7 @@ import io.restassured.RestAssured; import io.restassured.path.json.JsonPath; import io.restassured.response.Response; +import static jakarta.ws.rs.core.Response.Status.FORBIDDEN; import java.util.logging.Logger; import static org.hamcrest.CoreMatchers.equalTo; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -69,7 +70,15 @@ public void testCreateDeleteRoles() { body = addBuiltinRoleResponse.getBody().asString(); status = JsonPath.from(body).getString("status"); assertEquals("OK", status); - + + Response createNoPermsUser = UtilIT.createRandomUser(); + createNoPermsUser.prettyPrint(); + String noPermsapiToken = UtilIT.getApiTokenFromResponse(createNoPermsUser); + + Response noPermsResponse = UtilIT.viewDataverseRole("testRole", noPermsapiToken); + noPermsResponse.prettyPrint(); + noPermsResponse.then().assertThat().statusCode(FORBIDDEN.getStatusCode()); + Response viewDataverseRoleResponse = UtilIT.viewDataverseRole("testRole", apiToken); viewDataverseRoleResponse.prettyPrint(); body = viewDataverseRoleResponse.getBody().asString(); From f99d67e2b6bf9d3464a840ef8c5ee1fa5f96945e Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 20 Dec 2024 14:25:53 -0500 Subject: [PATCH 044/137] add change from 401 to 403 for /api/roles, improve docs #10340 --- doc/sphinx-guides/source/api/changelog.rst | 1 + doc/sphinx-guides/source/api/native-api.rst | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst index 162574e7799..5ae152aeace 100644 --- a/doc/sphinx-guides/source/api/changelog.rst +++ b/doc/sphinx-guides/source/api/changelog.rst @@ -11,6 +11,7 @@ v6.6 ---- - **/api/metadatablocks** is no longer returning duplicated metadata properties and does not omit metadata properties when called. +- **/api/roles**: :ref:`show-role` now properly returns 403 Forbidden instead of 401 Unauthorized when you pass a working API token that doesn't have the right permission. v6.5 ---- diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index dabca195e37..570b6308a64 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -4563,12 +4563,22 @@ Create Role Roles can be created globally (:ref:`create-global-role`) or for individual Dataverse collections (:ref:`create-role-in-collection`). +.. _show-role: + Show Role ~~~~~~~~~ -Shows the role with ``id``:: +You must have ``ManageDataversePermissions`` to be able to show a role that was created using :ref:`create-role-in-collection`. Global roles (:ref:`create-global-role`) only be shown with a superuser API token. + +A curl example using an ``ID``: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=11 - GET http://$SERVER/api/roles/$id + curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/roles/$ID" Delete Role ~~~~~~~~~~~ From a2cb8ae72c83b1d3be4e1d50148e1db4574639f5 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 20 Dec 2024 14:30:48 -0500 Subject: [PATCH 045/137] make release note more accurate, only one endpoint affected #10340 --- doc/release-notes/10340-forbidden.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/doc/release-notes/10340-forbidden.md b/doc/release-notes/10340-forbidden.md index 0910646c7bd..5997f717d64 100644 --- a/doc/release-notes/10340-forbidden.md +++ b/doc/release-notes/10340-forbidden.md @@ -1,7 +1,3 @@ -### API Now Returns 403 Forbidden for Permission Checks - -Dataverse was returning 401 Unauthorized when a permission check failed. This has been corrected to return 403 Forbidden in these cases. That is, the API token is known to be good (401 otherwise) but the user lacks permission (403 is now sent). See also #10340 and #11116. - ### Backward Incompatible Changes -See "API Now Returns 403 Forbidden for Permission Checks" above. +The [Show Role](https://dataverse-guide--11116.org.readthedocs.build/en/11116/api/native-api.html#show-role) API endpoint was returning 401 Unauthorized when a permission check failed. This has been corrected to return 403 Forbidden instead. That is, the API token is known to be good (401 otherwise) but the user lacks permission (403 is now sent). See also the [API Changelog](https://dataverse-guide--11116.org.readthedocs.build/en/11116/api/changelog.html), #10340, and #11116. From 28f54f36d7f570987f69468fae26fb7f69dee032 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 20 Dec 2024 23:57:12 +0000 Subject: [PATCH 046/137] Bump docker/setup-qemu-action from 2 to 3 Bumps [docker/setup-qemu-action](https://github.com/docker/setup-qemu-action) from 2 to 3. - [Release notes](https://github.com/docker/setup-qemu-action/releases) - [Commits](https://github.com/docker/setup-qemu-action/compare/v2...v3) --- updated-dependencies: - dependency-name: docker/setup-qemu-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/container_app_pr.yml | 2 +- .github/workflows/container_app_push.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/container_app_pr.yml b/.github/workflows/container_app_pr.yml index c3f9e7bdc0d..de4812e9b10 100644 --- a/.github/workflows/container_app_pr.yml +++ b/.github/workflows/container_app_pr.yml @@ -42,7 +42,7 @@ jobs: password: ${{ secrets.GHCR_TOKEN }} - name: Set up QEMU for multi-arch builds - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 # Get the image tag from either the command or default to branch name (Not used for now) #- name: Get the target tag name diff --git a/.github/workflows/container_app_push.yml b/.github/workflows/container_app_push.yml index afb4f6f874b..e9e93e1e001 100644 --- a/.github/workflows/container_app_push.yml +++ b/.github/workflows/container_app_push.yml @@ -139,7 +139,7 @@ jobs: password: ${{ secrets.GHCR_TOKEN }} - name: Set up QEMU for multi-arch builds - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Re-set image tag based on branch (if master) if: ${{ github.ref_name == 'master' }} From a50f624235e69c559bb2a00dcced373b63ced3f6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 20 Dec 2024 23:57:15 +0000 Subject: [PATCH 047/137] Bump appleboy/ssh-action from 1.0.0 to 1.2.0 Bumps [appleboy/ssh-action](https://github.com/appleboy/ssh-action) from 1.0.0 to 1.2.0. - [Release notes](https://github.com/appleboy/ssh-action/releases) - [Changelog](https://github.com/appleboy/ssh-action/blob/master/.goreleaser.yaml) - [Commits](https://github.com/appleboy/ssh-action/compare/v1.0.0...v1.2.0) --- updated-dependencies: - dependency-name: appleboy/ssh-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/deploy_beta_testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy_beta_testing.yml b/.github/workflows/deploy_beta_testing.yml index eca8416732a..ebf79275bec 100644 --- a/.github/workflows/deploy_beta_testing.yml +++ b/.github/workflows/deploy_beta_testing.yml @@ -68,7 +68,7 @@ jobs: overwrite: true - name: Execute payara war deployment remotely - uses: appleboy/ssh-action@v1.0.0 + uses: appleboy/ssh-action@v1.2.0 env: INPUT_WAR_FILE: ${{ env.war_file }} with: From 70563a38fc902ef59a2e646adad97db8366419b9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 20 Dec 2024 23:57:18 +0000 Subject: [PATCH 048/137] Bump docker/login-action from 2 to 3 Bumps [docker/login-action](https://github.com/docker/login-action) from 2 to 3. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/v2...v3) --- updated-dependencies: - dependency-name: docker/login-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/container_app_pr.yml | 2 +- .github/workflows/container_app_push.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/container_app_pr.yml b/.github/workflows/container_app_pr.yml index c3f9e7bdc0d..a011b972356 100644 --- a/.github/workflows/container_app_pr.yml +++ b/.github/workflows/container_app_pr.yml @@ -35,7 +35,7 @@ jobs: # Note: Accessing, pushing tags etc. to GHCR will only succeed in upstream because secrets. - name: Login to Github Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ secrets.GHCR_USERNAME }} diff --git a/.github/workflows/container_app_push.yml b/.github/workflows/container_app_push.yml index afb4f6f874b..fd308e037f9 100644 --- a/.github/workflows/container_app_push.yml +++ b/.github/workflows/container_app_push.yml @@ -126,13 +126,13 @@ jobs: # Depending on context, we push to different targets. Login accordingly. - if: github.event_name != 'pull_request' name: Log in to Docker Hub registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - if: ${{ github.event_name == 'pull_request' }} name: Login to Github Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ secrets.GHCR_USERNAME }} From 755fd4dcd283f960a52c7ee69a222a729e9cd72b Mon Sep 17 00:00:00 2001 From: GPortas Date: Mon, 23 Dec 2024 13:50:03 +0100 Subject: [PATCH 049/137] Stash: CreateDataverseFeaturedItemCommand WIP --- .../iq/dataverse/DataverseFeaturedItem.java | 22 ++++++++++++++ .../harvard/iq/dataverse/api/Dataverses.java | 4 ++- .../CreateDataverseFeaturedItemCommand.java | 29 +++++++++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java create mode 100644 src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java new file mode 100644 index 00000000000..014f323c1bd --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java @@ -0,0 +1,22 @@ +package edu.harvard.iq.dataverse; + +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; + +@Entity +public class DataverseFeaturedItem { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + public void setId(Long id) { + this.id = id; + } + + public Long getId() { + return id; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index e7ca7033210..2c934f92180 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -1734,6 +1734,7 @@ public Response getUserPermissionsOnDataverse(@Context ContainerRequestContext c return ok(jsonObjectBuilder); } + // TODO @PUT @AuthRequired @Consumes(MediaType.MULTIPART_FORM_DATA) @@ -1742,6 +1743,7 @@ public Response updateFeaturedItems(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf, @FormDataParam("title") String title, @FormDataParam("content") String content, + @FormDataParam("order") int order, @FormDataParam("file") InputStream fileInputStream, @FormDataParam("file") FormDataContentDisposition contentDispositionHeader) { Dataverse dataverse; @@ -1757,7 +1759,7 @@ public Response updateFeaturedItems(@Context ContainerRequestContext crc, uploadedFile.createNewFile(); } File file = FileUtil.inputStreamToFile(fileInputStream); - if (file.length() > systemConfig.getUploadLogoSizeLimit()) { + if (file.length() > 1000000) { return error(Response.Status.BAD_REQUEST, "File is larger than maximum size: " + systemConfig.getUploadLogoSizeLimit() + "."); } Files.copy(fileInputStream, uploadedFile.toPath(), StandardCopyOption.REPLACE_EXISTING); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java new file mode 100644 index 00000000000..7c3c36731dc --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java @@ -0,0 +1,29 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.DataverseFeaturedItem; +import edu.harvard.iq.dataverse.engine.command.AbstractCommand; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; + +/** + * A command that creates a featured item in a {@link Dataverse}. + */ +//TODO permissions +@RequiredPermissions({}) +public class CreateDataverseFeaturedItemCommand extends AbstractCommand { + + private final Dataverse dataverse; + + public CreateDataverseFeaturedItemCommand(DataverseRequest request, Dataverse dataverse) { + super(request, dataverse); + this.dataverse = dataverse; + } + + @Override + public DataverseFeaturedItem execute(CommandContext ctxt) throws CommandException { + return null; + } +} From 67e3fe0c03702dc82a80c8541c224aa03d3e0034 Mon Sep 17 00:00:00 2001 From: GPortas Date: Tue, 24 Dec 2024 19:04:10 +0100 Subject: [PATCH 050/137] Changed: multiple featured item creation endpoint converted to single --- .../iq/dataverse/DataverseFeaturedItem.java | 57 +++++++++++- .../harvard/iq/dataverse/api/Dataverses.java | 29 ++++++ .../CreateDataverseFeaturedItemCommand.java | 89 +++++++++++++++++-- .../iq/dataverse/settings/JvmSettings.java | 3 + .../iq/dataverse/util/json/JsonPrinter.java | 8 ++ src/main/java/propertyFiles/Bundle.properties | 3 + .../META-INF/microprofile-config.properties | 2 + .../iq/dataverse/api/DataversesIT.java | 16 +++- .../edu/harvard/iq/dataverse/api/UtilIT.java | 4 +- 9 files changed, 198 insertions(+), 13 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java index 014f323c1bd..700fe3f2320 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java @@ -1,17 +1,34 @@ package edu.harvard.iq.dataverse; -import jakarta.persistence.Entity; -import jakarta.persistence.GeneratedValue; -import jakarta.persistence.GenerationType; -import jakarta.persistence.Id; +import jakarta.persistence.*; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.Size; @Entity +@Table(indexes = {@Index(columnList = "dataverse_id"), @Index(columnList = "displayOrder")}) public class DataverseFeaturedItem { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) private Long id; + @ManyToOne + @JoinColumn(name = "dataverse_id", nullable = false) + private Dataverse dataverse; + + @NotBlank + @Size(max = 15000) + @Lob + @Column(columnDefinition = "TEXT", nullable = false) + private String content; + + @Min(0) + @Column(nullable = false) + private int displayOrder; + + private String imageFileName; + public void setId(Long id) { this.id = id; } @@ -19,4 +36,36 @@ public void setId(Long id) { public Long getId() { return id; } + + public Dataverse getDataverse() { + return dataverse; + } + + public void setDataverse(Dataverse dataverse) { + this.dataverse = dataverse; + } + + public String getContent() { + return content; + } + + public void setContent(String content) { + this.content = content; + } + + public int getDisplayOrder() { + return displayOrder; + } + + public void setDisplayOrder(int displayOrder) { + this.displayOrder = displayOrder; + } + + public String getImageFileName() { + return imageFileName; + } + + public void setImageFileName(String imageFileName) { + this.imageFileName = imageFileName; + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 2c934f92180..315d3279ba6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -1781,4 +1781,33 @@ private File createTempDir(Dataverse editDv) { throw new RuntimeException("Error creating temp directory", e); // improve error handling } } + + @POST + @AuthRequired + @Consumes(MediaType.MULTIPART_FORM_DATA) + @Path("{identifier}/featuredItem") + public Response createFeaturedItem(@Context ContainerRequestContext crc, + @PathParam("identifier") String dvIdtf, + @FormDataParam("content") String content, + @FormDataParam("order") int order, + @FormDataParam("file") InputStream fileInputStream, + @FormDataParam("file") FormDataContentDisposition contentDispositionHeader) { + Dataverse dataverse; + try { + dataverse = findDataverseOrDie(dvIdtf); + } catch (WrappedResponse wr) { + return wr.getResponse(); + } + NewDataverseFeaturedItemDTO newDataverseFeaturedItemDTO = NewDataverseFeaturedItemDTO.fromFormData(content, order, fileInputStream, contentDispositionHeader); + try { + DataverseFeaturedItem dataverseFeaturedItem = execCommand(new CreateDataverseFeaturedItemCommand( + createDataverseRequest(getRequestUser(crc)), + dataverse, + newDataverseFeaturedItemDTO + )); + return ok(json(dataverseFeaturedItem)); + } catch (WrappedResponse e) { + return e.getResponse(); + } + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java index 7c3c36731dc..fd29a5f9eab 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java @@ -2,28 +2,105 @@ import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.DataverseFeaturedItem; +import edu.harvard.iq.dataverse.api.dto.NewDataverseFeaturedItemDTO; import edu.harvard.iq.dataverse.engine.command.AbstractCommand; import edu.harvard.iq.dataverse.engine.command.CommandContext; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; +import edu.harvard.iq.dataverse.settings.JvmSettings; +import edu.harvard.iq.dataverse.util.BundleUtil; +import edu.harvard.iq.dataverse.util.FileUtil; +import org.apache.tika.Tika; -/** - * A command that creates a featured item in a {@link Dataverse}. - */ -//TODO permissions +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.List; + +// TODO: Permissions @RequiredPermissions({}) public class CreateDataverseFeaturedItemCommand extends AbstractCommand { private final Dataverse dataverse; + private final NewDataverseFeaturedItemDTO newDataverseFeaturedItemDTO; - public CreateDataverseFeaturedItemCommand(DataverseRequest request, Dataverse dataverse) { + public CreateDataverseFeaturedItemCommand(DataverseRequest request, Dataverse dataverse, NewDataverseFeaturedItemDTO newDataverseFeaturedItemDTO) { super(request, dataverse); this.dataverse = dataverse; + this.newDataverseFeaturedItemDTO = newDataverseFeaturedItemDTO; } @Override public DataverseFeaturedItem execute(CommandContext ctxt) throws CommandException { - return null; + DataverseFeaturedItem featuredItem = new DataverseFeaturedItem(); + setImageIfAvailable(featuredItem); + featuredItem.setContent(newDataverseFeaturedItemDTO.getContent()); + featuredItem.setDisplayOrder(newDataverseFeaturedItemDTO.getDisplayOrder()); + featuredItem.setDataverse(dataverse); + return featuredItem; + } + + private void setImageIfAvailable(DataverseFeaturedItem featuredItem) throws IllegalCommandException { + if (newDataverseFeaturedItemDTO.getImageFileName() != null) { + try { + prepareUploadedImageFile(); + } catch (IOException e) { + throw new RuntimeException(BundleUtil.getStringFromBundle("dataverse.create.featuredItem.error.imageFileProcessing", List.of(e.getMessage())), e); + } + featuredItem.setImageFileName(newDataverseFeaturedItemDTO.getImageFileName()); + } + } + + private void prepareUploadedImageFile() throws IOException, IllegalCommandException { + // Step 1: Create a temporary directory to store the uploaded image + Path tempDir = createTempDir(); + File uploadedFile = new File(tempDir.toFile(), newDataverseFeaturedItemDTO.getImageFileName()); + + if (!uploadedFile.exists()) { + uploadedFile.createNewFile(); + } + + // Step 2: Convert the InputStream into a temporary file for validation + File tempFile = FileUtil.inputStreamToFile(newDataverseFeaturedItemDTO.getFileInputStream()); + + // Step 3: Validate the uploaded file (type and size) + validateFile(tempFile); + + // Step 4: Copy the validated file to the final destination + Files.copy(tempFile.toPath(), uploadedFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + } + + private Path createTempDir() throws IOException { + Path tempRoot = Path.of(JvmSettings.DOCROOT_DIRECTORY.lookup(), JvmSettings.FEATURED_ITEMS_IMAGE_UPLOADS_DIRECTORY.lookup(), dataverse.getId().toString(), String.valueOf(newDataverseFeaturedItemDTO.getDisplayOrder())); + Files.createDirectories(tempRoot); + return tempRoot; + } + + private void validateFile(File file) throws IOException, IllegalCommandException { + validateFileType(file); + validateFileSize(file); + } + + private void validateFileType(File file) throws IOException, IllegalCommandException { + Tika tika = new Tika(); + String mimeType = tika.detect(file); + boolean isImageFile = mimeType != null && mimeType.startsWith("image/"); + if (!isImageFile) { + throw new IllegalCommandException( + BundleUtil.getStringFromBundle("dataverse.create.featuredItem.error.invalidFileType"), + this + ); + } + } + + private void validateFileSize(File file) throws IllegalCommandException { + Integer featuredItemsImageMaxSize = JvmSettings.FEATURED_ITEMS_IMAGE_MAXSIZE.lookup(Integer.class); + if (file.length() > featuredItemsImageMaxSize) { + throw new IllegalCommandException(BundleUtil.getStringFromBundle("dataverse.create.featuredItem.error.fileSizeExceedsLimit", List.of(featuredItemsImageMaxSize.toString())), this); + } } } diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java index d7eea970b8a..482c5ecbbb5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java @@ -52,6 +52,9 @@ public enum JvmSettings { GUESTBOOK_AT_REQUEST(SCOPE_FILES, "guestbook-at-request"), GLOBUS_CACHE_MAXAGE(SCOPE_FILES, "globus-cache-maxage"), GLOBUS_TASK_MONITORING_SERVER(SCOPE_FILES, "globus-monitoring-server"), + SCOPE_FEATURED_ITEMS(SCOPE_FILES, "featured-items"), + FEATURED_ITEMS_IMAGE_MAXSIZE(SCOPE_FEATURED_ITEMS, "image-maxsize"), + FEATURED_ITEMS_IMAGE_UPLOADS_DIRECTORY(SCOPE_FEATURED_ITEMS, "image-uploads"), //STORAGE DRIVER SETTINGS SCOPE_DRIVER(SCOPE_FILES), diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 91af13c79a3..8d6bf3a9bd2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -1417,4 +1417,12 @@ private static JsonObjectBuilder jsonDataverseInputLevel(DataverseFieldTypeInput jsonObjectBuilder.add("include", inputLevel.isInclude()); return jsonObjectBuilder; } + + public static JsonObjectBuilder json(DataverseFeaturedItem dataverseFeaturedItem) { + return jsonObjectBuilder() + .add("id", dataverseFeaturedItem.getId()) + .add("content", dataverseFeaturedItem.getContent()) + .add("imageFileName", dataverseFeaturedItem.getImageFileName()) + .add("displayOrder", dataverseFeaturedItem.getDisplayOrder()); + } } diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 193e4d2263f..750b1b4f429 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -979,6 +979,9 @@ dataverse.facets.error.fieldtypenotfacetable=Dataset field type '{0}' is not fac dataverse.metadatablocks.error.invalidmetadatablockname=Invalid metadata block name: {0} dataverse.create.error.jsonparse=Error parsing Json: {0} dataverse.create.error.jsonparsetodataverse=Error parsing the POSTed json into a dataverse: {0} +dataverse.create.featuredItem.error.imageFileProcessing=Error processing featured item file: {0} +dataverse.create.featuredItem.error.fileSizeExceedsLimit=File exceeds the maximum size of {0} +dataverse.create.featuredItem.error.invalidFileType=Invalid image file type # rolesAndPermissionsFragment.xhtml # advanced.xhtml diff --git a/src/main/resources/META-INF/microprofile-config.properties b/src/main/resources/META-INF/microprofile-config.properties index b0bc92cf975..95f30b6ba1d 100644 --- a/src/main/resources/META-INF/microprofile-config.properties +++ b/src/main/resources/META-INF/microprofile-config.properties @@ -19,6 +19,8 @@ dataverse.files.directory=${STORAGE_DIR:/tmp/dataverse} dataverse.files.uploads=${STORAGE_DIR:${com.sun.aas.instanceRoot}}/uploads dataverse.files.docroot=${STORAGE_DIR:${com.sun.aas.instanceRoot}}/docroot dataverse.files.globus-cache-maxage=5 +dataverse.files.featured-items.image-maxsize=1000000 +dataverse.files.featured-items.image-uploads=featuredItems # SEARCH INDEX dataverse.solr.host=localhost diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 53f9b1bf06d..40567ec2580 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -1568,8 +1568,22 @@ public void testUpdateFeaturedItems() { createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode()); String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + // Should not return any error when passing correct file and data String pathToTestFile = "src/test/resources/images/coffeeshop.png"; - Response updateFeatureItemsResponse = UtilIT.updateFeaturedItems(dataverseAlias, apiToken, "test", "test", pathToTestFile); + Response updateFeatureItemsResponse = UtilIT.createFeaturedItem(dataverseAlias, apiToken, "test", "test", pathToTestFile); + updateFeatureItemsResponse.then().assertThat() + .body("data.content", equalTo("test")) + .body("data.imageFileName", equalTo("coffeeshop.png")) + .body("data.displayOrder", equalTo(0)) + .statusCode(OK.getStatusCode()); + + // Should return error when passing incorrect file type + pathToTestFile = "src/test/resources/tab/test.tab"; + updateFeatureItemsResponse = UtilIT.createFeaturedItem(dataverseAlias, apiToken, "test", "test", pathToTestFile); updateFeatureItemsResponse.prettyPrint(); + updateFeatureItemsResponse.then().assertThat() + .body("message", equalTo(BundleUtil.getStringFromBundle("dataverse.create.featuredItem.error.invalidFileType"))) + // FIXME + .statusCode(BAD_REQUEST.getStatusCode()); } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index c7886124191..4101504db10 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -4305,7 +4305,7 @@ static Response deleteDatasetTypes(long doomed, String apiToken) { .delete("/api/datasets/datasetTypes/" + doomed); } - static Response updateFeaturedItems(String dataverseAlias, String apiToken, String title, String content, String pathToFile) { + static Response createFeaturedItem(String dataverseAlias, String apiToken, String title, String content, String pathToFile) { return given() .header(API_TOKEN_HTTP_HEADER, apiToken) .contentType(ContentType.MULTIPART) @@ -4313,6 +4313,6 @@ static Response updateFeaturedItems(String dataverseAlias, String apiToken, Stri .multiPart("content", content) .multiPart("file", new File(pathToFile)) .when() - .put("/api/dataverses/" + dataverseAlias + "/featuredItems"); + .post("/api/dataverses/" + dataverseAlias + "/featuredItem"); } } From b7352aa4df68a2de076e4bc274f20962da608f4a Mon Sep 17 00:00:00 2001 From: GPortas Date: Tue, 24 Dec 2024 19:05:27 +0100 Subject: [PATCH 051/137] Removed: unused/incorrect code --- .../harvard/iq/dataverse/api/Dataverses.java | 48 ------------------- 1 file changed, 48 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 315d3279ba6..ad074703e9c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -1734,54 +1734,6 @@ public Response getUserPermissionsOnDataverse(@Context ContainerRequestContext c return ok(jsonObjectBuilder); } - // TODO - @PUT - @AuthRequired - @Consumes(MediaType.MULTIPART_FORM_DATA) - @Path("{identifier}/featuredItems") - public Response updateFeaturedItems(@Context ContainerRequestContext crc, - @PathParam("identifier") String dvIdtf, - @FormDataParam("title") String title, - @FormDataParam("content") String content, - @FormDataParam("order") int order, - @FormDataParam("file") InputStream fileInputStream, - @FormDataParam("file") FormDataContentDisposition contentDispositionHeader) { - Dataverse dataverse; - try { - dataverse = findDataverseOrDie(dvIdtf); - } catch (WrappedResponse wr) { - return wr.getResponse(); - } - try { - String fileName = contentDispositionHeader.getFileName(); - File uploadedFile = new File(createTempDir(dataverse), fileName); - if (!uploadedFile.exists()) { - uploadedFile.createNewFile(); - } - File file = FileUtil.inputStreamToFile(fileInputStream); - if (file.length() > 1000000) { - return error(Response.Status.BAD_REQUEST, "File is larger than maximum size: " + systemConfig.getUploadLogoSizeLimit() + "."); - } - Files.copy(fileInputStream, uploadedFile.toPath(), StandardCopyOption.REPLACE_EXISTING); - } catch (IOException e) { - throw new RuntimeException(e); - } - return ok(""); - } - - private File createTempDir(Dataverse editDv) { - try { - // Create the temporary space if not yet existing (will silently ignore preexisting) - // Note that the docroot directory is checked within ConfigCheckService for presence and write access. - java.nio.file.Path tempRoot = java.nio.file.Path.of(JvmSettings.DOCROOT_DIRECTORY.lookup(), "featuredItems"); - Files.createDirectories(tempRoot); - - return Files.createTempDirectory(tempRoot, editDv.getId().toString()).toFile(); - } catch (IOException e) { - throw new RuntimeException("Error creating temp directory", e); // improve error handling - } - } - @POST @AuthRequired @Consumes(MediaType.MULTIPART_FORM_DATA) From 0744c306040792ac212b6df804ba8f72a4b03845 Mon Sep 17 00:00:00 2001 From: GPortas Date: Tue, 24 Dec 2024 19:06:12 +0100 Subject: [PATCH 052/137] Removed: unused imports --- src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index ad074703e9c..61dff7173fe 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -24,7 +24,6 @@ import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.ConstraintViolationUtil; -import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.StringUtil; import static edu.harvard.iq.dataverse.util.StringUtil.nonEmpty; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.*; @@ -35,8 +34,6 @@ import edu.harvard.iq.dataverse.util.json.JsonUtil; import java.io.*; -import java.nio.file.Files; -import java.nio.file.StandardCopyOption; import java.util.*; import java.util.logging.Level; import java.util.logging.Logger; From 2c96f21f55f4bc787611b80074fb9574db54bc7b Mon Sep 17 00:00:00 2001 From: GPortas Date: Thu, 26 Dec 2024 11:54:39 +0100 Subject: [PATCH 053/137] Added: persisting featured items in the database --- .../DataverseFeaturedItemServiceBean.java | 25 ++++++++++++ .../iq/dataverse/EjbDataverseEngine.java | 10 ++++- .../api/dto/NewDataverseFeaturedItemDTO.java | 40 +++++++++++++++++++ .../engine/command/CommandContext.java | 23 ++--------- .../CreateDataverseFeaturedItemCommand.java | 2 +- .../dataverse/engine/TestCommandContext.java | 5 +++ 6 files changed, 83 insertions(+), 22 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java create mode 100644 src/main/java/edu/harvard/iq/dataverse/api/dto/NewDataverseFeaturedItemDTO.java diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java new file mode 100644 index 00000000000..eb31923a360 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java @@ -0,0 +1,25 @@ +package edu.harvard.iq.dataverse; + +import jakarta.ejb.Stateless; +import jakarta.inject.Named; +import jakarta.persistence.EntityManager; +import jakarta.persistence.PersistenceContext; + +import java.io.Serializable; + +@Stateless +@Named +public class DataverseFeaturedItemServiceBean implements Serializable { + + @PersistenceContext(unitName = "VDCNet-ejbPU") + private EntityManager em; + + public DataverseFeaturedItem save(DataverseFeaturedItem dataverseFeaturedItem) { + if (dataverseFeaturedItem.getId() == null) { + em.persist(dataverseFeaturedItem); + return dataverseFeaturedItem; + } else { + return em.merge(dataverseFeaturedItem); + } + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java b/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java index 0561fed8a97..f8ba218d485 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java +++ b/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java @@ -184,7 +184,10 @@ public class EjbDataverseEngine { ConfirmEmailServiceBean confirmEmailService; @EJB - StorageUseServiceBean storageUseService; + StorageUseServiceBean storageUseService; + + @EJB + DataverseFeaturedItemServiceBean dataverseFeaturedItemServiceBean; @EJB EjbDataverseEngineInner innerEngine; @@ -522,6 +525,11 @@ public DatasetFieldServiceBean dsField() { return dsField; } + @Override + public DataverseFeaturedItemServiceBean dataverseFeaturedItems() { + return dataverseFeaturedItemServiceBean; + } + @Override public StorageUseServiceBean storageUse() { return storageUseService; diff --git a/src/main/java/edu/harvard/iq/dataverse/api/dto/NewDataverseFeaturedItemDTO.java b/src/main/java/edu/harvard/iq/dataverse/api/dto/NewDataverseFeaturedItemDTO.java new file mode 100644 index 00000000000..b81dd67aa97 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/api/dto/NewDataverseFeaturedItemDTO.java @@ -0,0 +1,40 @@ +package edu.harvard.iq.dataverse.api.dto; + +import org.glassfish.jersey.media.multipart.FormDataContentDisposition; + +import java.io.InputStream; + +public class NewDataverseFeaturedItemDTO { + private String content; + private int displayOrder; + private InputStream fileInputStream; + private String imageFileName; + + public static NewDataverseFeaturedItemDTO fromFormData(String content, + int order, + InputStream fileInputStream, + FormDataContentDisposition contentDispositionHeader) { + NewDataverseFeaturedItemDTO newDataverseFeaturedItemDTO = new NewDataverseFeaturedItemDTO(); + newDataverseFeaturedItemDTO.content = content; + newDataverseFeaturedItemDTO.displayOrder = order; + newDataverseFeaturedItemDTO.fileInputStream = fileInputStream; + newDataverseFeaturedItemDTO.imageFileName = contentDispositionHeader.getFileName(); + return newDataverseFeaturedItemDTO; + } + + public String getContent() { + return content; + } + + public int getDisplayOrder() { + return displayOrder; + } + + public InputStream getFileInputStream() { + return fileInputStream; + } + + public String getImageFileName() { + return imageFileName; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/CommandContext.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/CommandContext.java index 282cbb88988..b58f5f07ebc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/CommandContext.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/CommandContext.java @@ -1,28 +1,9 @@ package edu.harvard.iq.dataverse.engine.command; -import edu.harvard.iq.dataverse.DataFileServiceBean; -import edu.harvard.iq.dataverse.DatasetFieldServiceBean; -import edu.harvard.iq.dataverse.DatasetLinkingServiceBean; -import edu.harvard.iq.dataverse.DatasetServiceBean; -import edu.harvard.iq.dataverse.DatasetVersionServiceBean; -import edu.harvard.iq.dataverse.DataverseFacetServiceBean; -import edu.harvard.iq.dataverse.DataverseFieldTypeInputLevelServiceBean; -import edu.harvard.iq.dataverse.DataverseLinkingServiceBean; -import edu.harvard.iq.dataverse.DataverseRoleServiceBean; -import edu.harvard.iq.dataverse.DataverseServiceBean; +import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUserServiceBean; -import edu.harvard.iq.dataverse.DvObjectServiceBean; -import edu.harvard.iq.dataverse.FeaturedDataverseServiceBean; -import edu.harvard.iq.dataverse.FileDownloadServiceBean; -import edu.harvard.iq.dataverse.GuestbookResponseServiceBean; -import edu.harvard.iq.dataverse.GuestbookServiceBean; -import edu.harvard.iq.dataverse.MetadataBlockServiceBean; import edu.harvard.iq.dataverse.search.IndexServiceBean; -import edu.harvard.iq.dataverse.PermissionServiceBean; -import edu.harvard.iq.dataverse.RoleAssigneeServiceBean; import edu.harvard.iq.dataverse.search.SearchServiceBean; -import edu.harvard.iq.dataverse.TemplateServiceBean; -import edu.harvard.iq.dataverse.UserNotificationServiceBean; import edu.harvard.iq.dataverse.actionlogging.ActionLogServiceBean; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.groups.GroupServiceBean; @@ -152,4 +133,6 @@ public interface CommandContext { public void addCommand(Command command); public DatasetFieldServiceBean dsField(); + + public DataverseFeaturedItemServiceBean dataverseFeaturedItems(); } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java index fd29a5f9eab..ee496148ab5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java @@ -41,7 +41,7 @@ public DataverseFeaturedItem execute(CommandContext ctxt) throws CommandExceptio featuredItem.setContent(newDataverseFeaturedItemDTO.getContent()); featuredItem.setDisplayOrder(newDataverseFeaturedItemDTO.getDisplayOrder()); featuredItem.setDataverse(dataverse); - return featuredItem; + return ctxt.dataverseFeaturedItems().save(featuredItem); } private void setImageIfAvailable(DataverseFeaturedItem featuredItem) throws IllegalCommandException { diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/TestCommandContext.java b/src/test/java/edu/harvard/iq/dataverse/engine/TestCommandContext.java index b4b9c0d33f2..fd7e3f69bd2 100644 --- a/src/test/java/edu/harvard/iq/dataverse/engine/TestCommandContext.java +++ b/src/test/java/edu/harvard/iq/dataverse/engine/TestCommandContext.java @@ -246,6 +246,11 @@ public StorageUseServiceBean storageUse() { return null; } + @Override + public DataverseFeaturedItemServiceBean dataverseFeaturedItems() { + return null; + } + @Override public void beginCommandSequence() { throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. From 9cc611799a0fcab9b07cbf3cdcfc462d53093460 Mon Sep 17 00:00:00 2001 From: GPortas Date: Thu, 26 Dec 2024 12:06:30 +0100 Subject: [PATCH 054/137] Added: new InvalidCommandArgumentsException and using it in CreateDataverseFeaturedItemCommand --- .../iq/dataverse/api/AbstractApiBean.java | 7 +++--- .../InvalidCommandArgumentsException.java | 25 +++++++++++++++++++ .../CreateDataverseFeaturedItemCommand.java | 16 ++++++------ 3 files changed, 36 insertions(+), 12 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/engine/command/exception/InvalidCommandArgumentsException.java diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index 3257a3cc7ac..4efd161db53 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -14,14 +14,11 @@ import edu.harvard.iq.dataverse.dataset.DatasetTypeServiceBean; import edu.harvard.iq.dataverse.engine.command.Command; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; -import edu.harvard.iq.dataverse.engine.command.exception.CommandException; -import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; -import edu.harvard.iq.dataverse.engine.command.exception.PermissionException; +import edu.harvard.iq.dataverse.engine.command.exception.*; import edu.harvard.iq.dataverse.engine.command.impl.GetDraftDatasetVersionCommand; import edu.harvard.iq.dataverse.engine.command.impl.GetLatestAccessibleDatasetVersionCommand; import edu.harvard.iq.dataverse.engine.command.impl.GetLatestPublishedDatasetVersionCommand; import edu.harvard.iq.dataverse.engine.command.impl.GetSpecificPublishedDatasetVersionCommand; -import edu.harvard.iq.dataverse.engine.command.exception.RateLimitCommandException; import edu.harvard.iq.dataverse.externaltools.ExternalToolServiceBean; import edu.harvard.iq.dataverse.license.LicenseServiceBean; import edu.harvard.iq.dataverse.pidproviders.PidUtil; @@ -635,6 +632,8 @@ protected T execCommand( Command cmd ) throws WrappedResponse { throw new WrappedResponse(error(Response.Status.UNAUTHORIZED, "User " + cmd.getRequest().getUser().getIdentifier() + " is not permitted to perform requested action.") ); + } catch (InvalidCommandArgumentsException ex) { + throw new WrappedResponse(ex, error(Status.BAD_REQUEST, ex.getMessage())); } catch (CommandException ex) { Logger.getLogger(AbstractApiBean.class.getName()).log(Level.SEVERE, "Error while executing command " + cmd, ex); throw new WrappedResponse(ex, error(Status.INTERNAL_SERVER_ERROR, ex.getMessage())); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/exception/InvalidCommandArgumentsException.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/exception/InvalidCommandArgumentsException.java new file mode 100644 index 00000000000..95c6f52b880 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/exception/InvalidCommandArgumentsException.java @@ -0,0 +1,25 @@ +package edu.harvard.iq.dataverse.engine.command.exception; + +import edu.harvard.iq.dataverse.engine.command.Command; + +/** + * Exception thrown when a {@link Command} is executed with invalid or malformed arguments. + *

+ * This exception typically indicates that the input parameters provided to the command + * do not meet the required criteria (e.g., missing fields, invalid formats, or other + * constraints). + *

+ *

+ * Example scenarios: + *

    + *
  • A required argument is null or missing.
    • + *
  • An argument is in an invalid format (e.g., a malformed email address).
    • + *
  • Arguments violate business rules or constraints.
    • + *
+ */ +public class InvalidCommandArgumentsException extends CommandException { + + public InvalidCommandArgumentsException(String message, Command aCommand) { + super(message, aCommand); + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java index ee496148ab5..36506c67e43 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java @@ -8,7 +8,7 @@ import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; -import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; +import edu.harvard.iq.dataverse.engine.command.exception.InvalidCommandArgumentsException; import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.FileUtil; @@ -44,7 +44,7 @@ public DataverseFeaturedItem execute(CommandContext ctxt) throws CommandExceptio return ctxt.dataverseFeaturedItems().save(featuredItem); } - private void setImageIfAvailable(DataverseFeaturedItem featuredItem) throws IllegalCommandException { + private void setImageIfAvailable(DataverseFeaturedItem featuredItem) throws InvalidCommandArgumentsException { if (newDataverseFeaturedItemDTO.getImageFileName() != null) { try { prepareUploadedImageFile(); @@ -55,7 +55,7 @@ private void setImageIfAvailable(DataverseFeaturedItem featuredItem) throws Ille } } - private void prepareUploadedImageFile() throws IOException, IllegalCommandException { + private void prepareUploadedImageFile() throws IOException, InvalidCommandArgumentsException { // Step 1: Create a temporary directory to store the uploaded image Path tempDir = createTempDir(); File uploadedFile = new File(tempDir.toFile(), newDataverseFeaturedItemDTO.getImageFileName()); @@ -80,27 +80,27 @@ private Path createTempDir() throws IOException { return tempRoot; } - private void validateFile(File file) throws IOException, IllegalCommandException { + private void validateFile(File file) throws IOException, InvalidCommandArgumentsException { validateFileType(file); validateFileSize(file); } - private void validateFileType(File file) throws IOException, IllegalCommandException { + private void validateFileType(File file) throws IOException, InvalidCommandArgumentsException { Tika tika = new Tika(); String mimeType = tika.detect(file); boolean isImageFile = mimeType != null && mimeType.startsWith("image/"); if (!isImageFile) { - throw new IllegalCommandException( + throw new InvalidCommandArgumentsException( BundleUtil.getStringFromBundle("dataverse.create.featuredItem.error.invalidFileType"), this ); } } - private void validateFileSize(File file) throws IllegalCommandException { + private void validateFileSize(File file) throws InvalidCommandArgumentsException { Integer featuredItemsImageMaxSize = JvmSettings.FEATURED_ITEMS_IMAGE_MAXSIZE.lookup(Integer.class); if (file.length() > featuredItemsImageMaxSize) { - throw new IllegalCommandException(BundleUtil.getStringFromBundle("dataverse.create.featuredItem.error.fileSizeExceedsLimit", List.of(featuredItemsImageMaxSize.toString())), this); + throw new InvalidCommandArgumentsException(BundleUtil.getStringFromBundle("dataverse.create.featuredItem.error.fileSizeExceedsLimit", List.of(featuredItemsImageMaxSize.toString())), this); } } } From 11a05925233dab975d20168e4a39ede1e6924563 Mon Sep 17 00:00:00 2001 From: GPortas Date: Thu, 26 Dec 2024 16:14:37 +0100 Subject: [PATCH 055/137] Added: endpoint for listing featured items --- .../edu/harvard/iq/dataverse/Dataverse.java | 7 ++++ .../harvard/iq/dataverse/api/Dataverses.java | 13 +++++++ .../CreateDataverseFeaturedItemCommand.java | 4 +- .../ListDataverseFeaturedItemsCommand.java | 39 +++++++++++++++++++ .../iq/dataverse/util/json/JsonPrinter.java | 8 ++++ .../iq/dataverse/api/DataversesIT.java | 8 ++-- .../edu/harvard/iq/dataverse/api/UtilIT.java | 7 ++++ 7 files changed, 81 insertions(+), 5 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListDataverseFeaturedItemsCommand.java diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java index 1f11725e581..cf6c763d8fc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java +++ b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java @@ -351,6 +351,13 @@ public void setMetadataBlockFacets(List metadataBlo this.metadataBlockFacets = metadataBlockFacets; } + @OneToMany(mappedBy = "dataverse") + private List dataverseFeaturedItems = new ArrayList<>(); + + public List getDataverseFeaturedItems() { + return this.dataverseFeaturedItems; + } + public List getParentGuestbooks() { List retList = new ArrayList<>(); Dataverse testDV = this; diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 61dff7173fe..b2f174d9449 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -1759,4 +1759,17 @@ public Response createFeaturedItem(@Context ContainerRequestContext crc, return e.getResponse(); } } + + @GET + @AuthRequired + @Path("{identifier}/featuredItems") + public Response listFeaturedItems(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf) { + try { + Dataverse dataverse = findDataverseOrDie(dvIdtf); + List featuredItems = execCommand(new ListDataverseFeaturedItemsCommand(createDataverseRequest(getRequestUser(crc)), dataverse)); + return ok(jsonDataverseFeaturedItems(featuredItems)); + } catch (WrappedResponse e) { + return e.getResponse(); + } + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java index 36506c67e43..3057824b937 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java @@ -3,6 +3,7 @@ import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.DataverseFeaturedItem; import edu.harvard.iq.dataverse.api.dto.NewDataverseFeaturedItemDTO; +import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.engine.command.AbstractCommand; import edu.harvard.iq.dataverse.engine.command.CommandContext; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; @@ -21,8 +22,7 @@ import java.nio.file.StandardCopyOption; import java.util.List; -// TODO: Permissions -@RequiredPermissions({}) +@RequiredPermissions({Permission.EditDataverse}) public class CreateDataverseFeaturedItemCommand extends AbstractCommand { private final Dataverse dataverse; diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListDataverseFeaturedItemsCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListDataverseFeaturedItemsCommand.java new file mode 100644 index 00000000000..d8d36352e79 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListDataverseFeaturedItemsCommand.java @@ -0,0 +1,39 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.DataverseFeaturedItem; +import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.engine.command.AbstractCommand; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * List the featured items {@link DataverseFeaturedItem} of a {@link Dataverse}. + */ +public class ListDataverseFeaturedItemsCommand extends AbstractCommand> { + + private final Dataverse dataverse; + + public ListDataverseFeaturedItemsCommand(DataverseRequest request, Dataverse dataverse) { + super(request, dataverse); + this.dataverse = dataverse; + } + + @Override + public List execute(CommandContext ctxt) throws CommandException { + return dataverse.getDataverseFeaturedItems(); + } + + @Override + public Map> getRequiredPermissions() { + return Collections.singletonMap("", + dataverse.isReleased() ? Collections.emptySet() + : Collections.singleton(Permission.ViewUnpublishedDataverse)); + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 8d6bf3a9bd2..a4b5faa1e12 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -1418,6 +1418,14 @@ private static JsonObjectBuilder jsonDataverseInputLevel(DataverseFieldTypeInput return jsonObjectBuilder; } + public static JsonArrayBuilder jsonDataverseFeaturedItems(List dataverseFeaturedItems) { + JsonArrayBuilder featuredItemsArrayBuilder = Json.createArrayBuilder(); + for (DataverseFeaturedItem dataverseFeaturedItem : dataverseFeaturedItems) { + featuredItemsArrayBuilder.add(json(dataverseFeaturedItem)); + } + return featuredItemsArrayBuilder; + } + public static JsonObjectBuilder json(DataverseFeaturedItem dataverseFeaturedItem) { return jsonObjectBuilder() .add("id", dataverseFeaturedItem.getId()) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 40567ec2580..72205022b8c 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -1561,7 +1561,7 @@ public void testGetUserPermissionsOnDataverse() { } @Test - public void testUpdateFeaturedItems() { + public void testCreateFeaturedItem() { Response createUserResponse = UtilIT.createRandomUser(); String apiToken = UtilIT.getApiTokenFromResponse(createUserResponse); Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); @@ -1577,13 +1577,15 @@ public void testUpdateFeaturedItems() { .body("data.displayOrder", equalTo(0)) .statusCode(OK.getStatusCode()); + // TODO + Response listFeaturedItemsResponse = UtilIT.listDataverseFeaturedItems(dataverseAlias, apiToken); + listFeaturedItemsResponse.prettyPrint(); + // Should return error when passing incorrect file type pathToTestFile = "src/test/resources/tab/test.tab"; updateFeatureItemsResponse = UtilIT.createFeaturedItem(dataverseAlias, apiToken, "test", "test", pathToTestFile); - updateFeatureItemsResponse.prettyPrint(); updateFeatureItemsResponse.then().assertThat() .body("message", equalTo(BundleUtil.getStringFromBundle("dataverse.create.featuredItem.error.invalidFileType"))) - // FIXME .statusCode(BAD_REQUEST.getStatusCode()); } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 4101504db10..b9947e2e870 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -4315,4 +4315,11 @@ static Response createFeaturedItem(String dataverseAlias, String apiToken, Strin .when() .post("/api/dataverses/" + dataverseAlias + "/featuredItem"); } + + static Response listDataverseFeaturedItems(String dataverseAlias, String apiToken) { + return given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .contentType("application/json") + .get("/api/dataverses/" + dataverseAlias + "/featuredItems"); + } } From ec3b6f2d741008e191d062d8305c332af6dfb72c Mon Sep 17 00:00:00 2001 From: GPortas Date: Fri, 27 Dec 2024 18:56:31 +0000 Subject: [PATCH 056/137] Added: returning imageUrl with new associated endpoint for retrieving the featured item image from the client --- .../iq/dataverse/DataverseFeaturedItem.java | 5 ++ .../DataverseFeaturedItemServiceBean.java | 17 ++++++ .../edu/harvard/iq/dataverse/api/Access.java | 60 +++++++------------ .../CreateDataverseFeaturedItemCommand.java | 14 ++--- .../impl/GetDataverseFeaturedItemCommand.java | 37 ++++++++++++ .../ListDataverseFeaturedItemsCommand.java | 2 +- .../iq/dataverse/util/json/JsonPrinter.java | 1 + 7 files changed, 91 insertions(+), 45 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDataverseFeaturedItemCommand.java diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java index 700fe3f2320..48a2fc335fa 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java @@ -1,5 +1,6 @@ package edu.harvard.iq.dataverse; +import edu.harvard.iq.dataverse.util.SystemConfig; import jakarta.persistence.*; import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.Min; @@ -68,4 +69,8 @@ public String getImageFileName() { public void setImageFileName(String imageFileName) { this.imageFileName = imageFileName; } + + public String getImageFileUrl() { + return SystemConfig.getDataverseSiteUrlStatic() + "/api/access/dataverseFeatureItemImage/" + id; + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java index eb31923a360..21bf9b82b0f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java @@ -1,11 +1,16 @@ package edu.harvard.iq.dataverse; +import edu.harvard.iq.dataverse.settings.JvmSettings; import jakarta.ejb.Stateless; import jakarta.inject.Named; import jakarta.persistence.EntityManager; import jakarta.persistence.PersistenceContext; +import java.io.IOException; +import java.io.InputStream; import java.io.Serializable; +import java.nio.file.Files; +import java.nio.file.Path; @Stateless @Named @@ -14,6 +19,10 @@ public class DataverseFeaturedItemServiceBean implements Serializable { @PersistenceContext(unitName = "VDCNet-ejbPU") private EntityManager em; + public DataverseFeaturedItem findById(Long id) { + return em.find(DataverseFeaturedItem.class, id); + } + public DataverseFeaturedItem save(DataverseFeaturedItem dataverseFeaturedItem) { if (dataverseFeaturedItem.getId() == null) { em.persist(dataverseFeaturedItem); @@ -22,4 +31,12 @@ public DataverseFeaturedItem save(DataverseFeaturedItem dataverseFeaturedItem) { return em.merge(dataverseFeaturedItem); } } + + public InputStream getImageFileAsInputStream(DataverseFeaturedItem dataverseFeaturedItem) throws IOException { + Path imagePath = Path.of(JvmSettings.DOCROOT_DIRECTORY.lookup(), + JvmSettings.FEATURED_ITEMS_IMAGE_UPLOADS_DIRECTORY.lookup(), + dataverseFeaturedItem.getDataverse().getId().toString(), + dataverseFeaturedItem.getImageFileName()); + return Files.newInputStream(imagePath); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index 16ac884180b..d970c5eed44 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -6,32 +6,7 @@ package edu.harvard.iq.dataverse.api; -import edu.harvard.iq.dataverse.AuxiliaryFile; -import edu.harvard.iq.dataverse.AuxiliaryFileServiceBean; -import edu.harvard.iq.dataverse.DataCitation; -import edu.harvard.iq.dataverse.DataFile; -import edu.harvard.iq.dataverse.FileAccessRequest; -import edu.harvard.iq.dataverse.FileMetadata; -import edu.harvard.iq.dataverse.DataFileServiceBean; -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetVersion; -import edu.harvard.iq.dataverse.DatasetVersionServiceBean; -import edu.harvard.iq.dataverse.DatasetServiceBean; -import edu.harvard.iq.dataverse.Dataverse; -import edu.harvard.iq.dataverse.DataverseRequestServiceBean; -import edu.harvard.iq.dataverse.DataverseRoleServiceBean; -import edu.harvard.iq.dataverse.DataverseServiceBean; -import edu.harvard.iq.dataverse.DataverseSession; -import edu.harvard.iq.dataverse.DataverseTheme; -import edu.harvard.iq.dataverse.FileDownloadServiceBean; -import edu.harvard.iq.dataverse.GuestbookResponse; -import edu.harvard.iq.dataverse.GuestbookResponseServiceBean; -import edu.harvard.iq.dataverse.PermissionServiceBean; -import edu.harvard.iq.dataverse.PermissionsWrapper; -import edu.harvard.iq.dataverse.RoleAssignment; -import edu.harvard.iq.dataverse.UserNotification; -import edu.harvard.iq.dataverse.UserNotificationServiceBean; -import edu.harvard.iq.dataverse.ThemeWidgetFragment; +import edu.harvard.iq.dataverse.*; import static edu.harvard.iq.dataverse.api.Datasets.handleVersion; @@ -55,15 +30,7 @@ import edu.harvard.iq.dataverse.engine.command.Command; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; -import edu.harvard.iq.dataverse.engine.command.impl.AssignRoleCommand; -import edu.harvard.iq.dataverse.engine.command.impl.GetDatasetCommand; -import edu.harvard.iq.dataverse.engine.command.impl.GetDraftDatasetVersionCommand; -import edu.harvard.iq.dataverse.engine.command.impl.GetLatestAccessibleDatasetVersionCommand; -import edu.harvard.iq.dataverse.engine.command.impl.GetLatestPublishedDatasetVersionCommand; -import edu.harvard.iq.dataverse.engine.command.impl.GetSpecificPublishedDatasetVersionCommand; -import edu.harvard.iq.dataverse.engine.command.impl.RequestAccessCommand; -import edu.harvard.iq.dataverse.engine.command.impl.RevokeRoleCommand; -import edu.harvard.iq.dataverse.engine.command.impl.UpdateDatasetVersionCommand; +import edu.harvard.iq.dataverse.engine.command.impl.*; import edu.harvard.iq.dataverse.export.DDIExportServiceBean; import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean; import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean.MakeDataCountEntry; @@ -88,7 +55,6 @@ import java.util.Arrays; import java.util.Date; import java.util.List; -import java.util.Properties; import java.util.logging.Level; import jakarta.inject.Inject; import jakarta.json.Json; @@ -133,7 +99,6 @@ import org.eclipse.microprofile.openapi.annotations.Operation; import org.eclipse.microprofile.openapi.annotations.media.Content; -import org.eclipse.microprofile.openapi.annotations.media.Schema; import org.eclipse.microprofile.openapi.annotations.parameters.RequestBody; import org.eclipse.microprofile.openapi.annotations.responses.APIResponse; import org.eclipse.microprofile.openapi.annotations.responses.APIResponses; @@ -199,6 +164,8 @@ public class Access extends AbstractApiBean { PermissionsWrapper permissionsWrapper; @Inject MakeDataCountLoggingServiceBean mdcLogService; + @Inject + DataverseFeaturedItemServiceBean dataverseFeaturedItemServiceBean; //@EJB @@ -2015,4 +1982,23 @@ private URI handleCustomZipDownload(User user, String customZipServiceUrl, Strin } return redirectUri; } + + @Path("dataverseFeatureItemImage/{itemId}") + @GET + @Produces({"image/png"}) + public InputStream getDataverseFeatureItemImage(@Context ContainerRequestContext crc, @PathParam("itemId") Long itemId) { + DataverseFeaturedItem dataverseFeaturedItem; + try { + dataverseFeaturedItem = execCommand(new GetDataverseFeaturedItemCommand(createDataverseRequest(getRequestUser(crc)), dataverseFeaturedItemServiceBean.findById(itemId))); + } catch (WrappedResponse wr) { + logger.warning("Cannot locate a dataverse featured item with id " + itemId); + return null; + } + try { + return dataverseFeaturedItemServiceBean.getImageFileAsInputStream(dataverseFeaturedItem); + } catch (IOException e) { + logger.warning("Error while obtaining the input stream for the image file associated with the dataverse featured item with id " + itemId); + return null; + } + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java index 3057824b937..074e6f34743 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java @@ -56,9 +56,9 @@ private void setImageIfAvailable(DataverseFeaturedItem featuredItem) throws Inva } private void prepareUploadedImageFile() throws IOException, InvalidCommandArgumentsException { - // Step 1: Create a temporary directory to store the uploaded image - Path tempDir = createTempDir(); - File uploadedFile = new File(tempDir.toFile(), newDataverseFeaturedItemDTO.getImageFileName()); + // Step 1: Create a directory to store the uploaded image + Path imageDir = createImageDir(); + File uploadedFile = new File(imageDir.toFile(), newDataverseFeaturedItemDTO.getImageFileName()); if (!uploadedFile.exists()) { uploadedFile.createNewFile(); @@ -74,10 +74,10 @@ private void prepareUploadedImageFile() throws IOException, InvalidCommandArgume Files.copy(tempFile.toPath(), uploadedFile.toPath(), StandardCopyOption.REPLACE_EXISTING); } - private Path createTempDir() throws IOException { - Path tempRoot = Path.of(JvmSettings.DOCROOT_DIRECTORY.lookup(), JvmSettings.FEATURED_ITEMS_IMAGE_UPLOADS_DIRECTORY.lookup(), dataverse.getId().toString(), String.valueOf(newDataverseFeaturedItemDTO.getDisplayOrder())); - Files.createDirectories(tempRoot); - return tempRoot; + private Path createImageDir() throws IOException { + Path imagePath = Path.of(JvmSettings.DOCROOT_DIRECTORY.lookup(), JvmSettings.FEATURED_ITEMS_IMAGE_UPLOADS_DIRECTORY.lookup(), dataverse.getId().toString()); + Files.createDirectories(imagePath); + return imagePath; } private void validateFile(File file) throws IOException, InvalidCommandArgumentsException { diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDataverseFeaturedItemCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDataverseFeaturedItemCommand.java new file mode 100644 index 00000000000..722c8050dbd --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDataverseFeaturedItemCommand.java @@ -0,0 +1,37 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.DataverseFeaturedItem; +import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.engine.command.AbstractCommand; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; + +import java.util.Collections; +import java.util.Map; +import java.util.Set; + +/** + * Retrieves a particular featured item {@link DataverseFeaturedItem}. + */ +public class GetDataverseFeaturedItemCommand extends AbstractCommand { + + private final DataverseFeaturedItem dataverseFeaturedItem; + + public GetDataverseFeaturedItemCommand(DataverseRequest request, DataverseFeaturedItem dataverseFeaturedItem) { + super(request, dataverseFeaturedItem.getDataverse()); + this.dataverseFeaturedItem = dataverseFeaturedItem; + } + + @Override + public DataverseFeaturedItem execute(CommandContext ctxt) throws CommandException { + return dataverseFeaturedItem; + } + + @Override + public Map> getRequiredPermissions() { + return Collections.singletonMap("", + dataverseFeaturedItem.getDataverse().isReleased() ? Collections.emptySet() + : Collections.singleton(Permission.ViewUnpublishedDataverse)); + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListDataverseFeaturedItemsCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListDataverseFeaturedItemsCommand.java index d8d36352e79..9ff2d2e1e71 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListDataverseFeaturedItemsCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListDataverseFeaturedItemsCommand.java @@ -14,7 +14,7 @@ import java.util.Set; /** - * List the featured items {@link DataverseFeaturedItem} of a {@link Dataverse}. + * Lists the featured items {@link DataverseFeaturedItem} of a {@link Dataverse}. */ public class ListDataverseFeaturedItemsCommand extends AbstractCommand> { diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index a4b5faa1e12..4f5b2898eff 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -1431,6 +1431,7 @@ public static JsonObjectBuilder json(DataverseFeaturedItem dataverseFeaturedItem .add("id", dataverseFeaturedItem.getId()) .add("content", dataverseFeaturedItem.getContent()) .add("imageFileName", dataverseFeaturedItem.getImageFileName()) + .add("imageFileUrl", dataverseFeaturedItem.getImageFileUrl()) .add("displayOrder", dataverseFeaturedItem.getDisplayOrder()); } } From 160ffa592e09d4fc17c4a22caf0e17012628bf83 Mon Sep 17 00:00:00 2001 From: GPortas Date: Mon, 30 Dec 2024 08:36:02 +0000 Subject: [PATCH 057/137] Removed: wrongly committed files --- ...arvard.iq.dataverse.pidproviders.PidProviderFactory | 6 ------ .../META-INF/services/io.gdcc.spi.export.Exporter | 10 ---------- 2 files changed, 16 deletions(-) delete mode 100644 src/main/resources/META-INF/services/edu.harvard.iq.dataverse.pidproviders.PidProviderFactory delete mode 100644 src/main/resources/META-INF/services/io.gdcc.spi.export.Exporter diff --git a/src/main/resources/META-INF/services/edu.harvard.iq.dataverse.pidproviders.PidProviderFactory b/src/main/resources/META-INF/services/edu.harvard.iq.dataverse.pidproviders.PidProviderFactory deleted file mode 100644 index acdfd927e45..00000000000 --- a/src/main/resources/META-INF/services/edu.harvard.iq.dataverse.pidproviders.PidProviderFactory +++ /dev/null @@ -1,6 +0,0 @@ -edu.harvard.iq.dataverse.pidproviders.doi.crossref.CrossRefDOIProviderFactory -edu.harvard.iq.dataverse.pidproviders.doi.datacite.DataCiteProviderFactory -edu.harvard.iq.dataverse.pidproviders.doi.ezid.EZIdProviderFactory -edu.harvard.iq.dataverse.pidproviders.doi.fake.FakeProviderFactory -edu.harvard.iq.dataverse.pidproviders.handle.HandleProviderFactory -edu.harvard.iq.dataverse.pidproviders.perma.PermaLinkProviderFactory diff --git a/src/main/resources/META-INF/services/io.gdcc.spi.export.Exporter b/src/main/resources/META-INF/services/io.gdcc.spi.export.Exporter deleted file mode 100644 index 873cea58911..00000000000 --- a/src/main/resources/META-INF/services/io.gdcc.spi.export.Exporter +++ /dev/null @@ -1,10 +0,0 @@ -edu.harvard.iq.dataverse.export.DCTermsExporter -edu.harvard.iq.dataverse.export.DDIExporter -edu.harvard.iq.dataverse.export.DataCiteExporter -edu.harvard.iq.dataverse.export.DublinCoreExporter -edu.harvard.iq.dataverse.export.HtmlCodeBookExporter -edu.harvard.iq.dataverse.export.JSONExporter -edu.harvard.iq.dataverse.export.OAI_DDIExporter -edu.harvard.iq.dataverse.export.OAI_OREExporter -edu.harvard.iq.dataverse.export.OpenAireExporter -edu.harvard.iq.dataverse.export.SchemaDotOrgExporter From f124f4a7c912d3bc2d965e0eeba85f9a1456669a Mon Sep 17 00:00:00 2001 From: GPortas Date: Mon, 30 Dec 2024 11:43:14 +0000 Subject: [PATCH 058/137] Stash: relocating file-specific logic and unit tests in progress --- .../iq/dataverse/DataverseFeaturedItem.java | 8 +- .../DataverseFeaturedItemServiceBean.java | 56 ++++++++ .../api/dto/NewDataverseFeaturedItemDTO.java | 16 +++ .../CreateDataverseFeaturedItemCommand.java | 84 ++++-------- ...reateDataverseFeaturedItemCommandTest.java | 120 ++++++++++++++++++ 5 files changed, 218 insertions(+), 66 deletions(-) create mode 100644 src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommandTest.java diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java index 48a2fc335fa..ef2421623f3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java @@ -30,14 +30,14 @@ public class DataverseFeaturedItem { private String imageFileName; - public void setId(Long id) { - this.id = id; - } - public Long getId() { return id; } + public void setId(Long id) { + this.id = id; + } + public Dataverse getDataverse() { return dataverse; } diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java index 21bf9b82b0f..1c71f667b5c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java @@ -1,16 +1,22 @@ package edu.harvard.iq.dataverse; import edu.harvard.iq.dataverse.settings.JvmSettings; +import edu.harvard.iq.dataverse.util.BundleUtil; +import edu.harvard.iq.dataverse.util.FileUtil; import jakarta.ejb.Stateless; import jakarta.inject.Named; import jakarta.persistence.EntityManager; import jakarta.persistence.PersistenceContext; +import org.apache.tika.Tika; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.Serializable; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.List; @Stateless @Named @@ -39,4 +45,54 @@ public InputStream getImageFileAsInputStream(DataverseFeaturedItem dataverseFeat dataverseFeaturedItem.getImageFileName()); return Files.newInputStream(imagePath); } + + public void saveDataverseFeaturedItemImageFile(InputStream inputStream, String imageFileName, Long dataverseId) throws IOException { + File tempFile = FileUtil.inputStreamToFile(inputStream); + validateImageFile(tempFile); + + Path imageDir = createDataverseFeaturedItemImageDir(dataverseId); + File uploadedFile = new File(imageDir.toFile(), imageFileName); + + if (!uploadedFile.exists()) { + uploadedFile.createNewFile(); + } + + Files.copy(tempFile.toPath(), uploadedFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + } + + // TODO: Move file-specific logic out of here + + private Path createDataverseFeaturedItemImageDir(Long dataverseId) throws IOException { + Path imagePath = Path.of(JvmSettings.DOCROOT_DIRECTORY.lookup(), JvmSettings.FEATURED_ITEMS_IMAGE_UPLOADS_DIRECTORY.lookup(), dataverseId.toString()); + Files.createDirectories(imagePath); + return imagePath; + } + + private void validateImageFile(File file) throws IOException, IllegalArgumentException { + if (!isValidDataverseFeaturedItemFileType(file)) { + throw new IllegalArgumentException( + BundleUtil.getStringFromBundle("dataverse.create.featuredItem.error.invalidFileType") + ); + } + if (!isValidDataverseFeaturedItemFileSize(file)) { + String maxAllowedSize = getMaxAllowedDataverseFeaturedItemFileSize().toString(); + throw new IllegalArgumentException( + BundleUtil.getStringFromBundle("dataverse.create.featuredItem.error.fileSizeExceedsLimit", List.of(maxAllowedSize)) + ); + } + } + + private boolean isValidDataverseFeaturedItemFileType(File file) throws IOException { + Tika tika = new Tika(); + String mimeType = tika.detect(file); + return mimeType != null && mimeType.startsWith("image/"); + } + + private boolean isValidDataverseFeaturedItemFileSize(File file) { + return file.length() <= getMaxAllowedDataverseFeaturedItemFileSize(); + } + + private Integer getMaxAllowedDataverseFeaturedItemFileSize() { + return JvmSettings.FEATURED_ITEMS_IMAGE_MAXSIZE.lookup(Integer.class); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/dto/NewDataverseFeaturedItemDTO.java b/src/main/java/edu/harvard/iq/dataverse/api/dto/NewDataverseFeaturedItemDTO.java index b81dd67aa97..e1f5190cb41 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/dto/NewDataverseFeaturedItemDTO.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/dto/NewDataverseFeaturedItemDTO.java @@ -22,18 +22,34 @@ public static NewDataverseFeaturedItemDTO fromFormData(String content, return newDataverseFeaturedItemDTO; } + public void setContent(String content) { + this.content = content; + } + public String getContent() { return content; } + public void setDisplayOrder(int displayOrder) { + this.displayOrder = displayOrder; + } + public int getDisplayOrder() { return displayOrder; } + public void setFileInputStream(InputStream fileInputStream) { + this.fileInputStream = fileInputStream; + } + public InputStream getFileInputStream() { return fileInputStream; } + public void setImageFileName(String imageFileName) { + this.imageFileName = imageFileName; + } + public String getImageFileName() { return imageFileName; } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java index 074e6f34743..c79f36c10fa 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java @@ -10,16 +10,9 @@ import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.InvalidCommandArgumentsException; -import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.util.BundleUtil; -import edu.harvard.iq.dataverse.util.FileUtil; -import org.apache.tika.Tika; -import java.io.File; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; import java.util.List; @RequiredPermissions({Permission.EditDataverse}) @@ -37,70 +30,37 @@ public CreateDataverseFeaturedItemCommand(DataverseRequest request, Dataverse da @Override public DataverseFeaturedItem execute(CommandContext ctxt) throws CommandException { DataverseFeaturedItem featuredItem = new DataverseFeaturedItem(); - setImageIfAvailable(featuredItem); + + setImageIfAvailable(featuredItem, ctxt); + featuredItem.setContent(newDataverseFeaturedItemDTO.getContent()); featuredItem.setDisplayOrder(newDataverseFeaturedItemDTO.getDisplayOrder()); featuredItem.setDataverse(dataverse); + return ctxt.dataverseFeaturedItems().save(featuredItem); } - private void setImageIfAvailable(DataverseFeaturedItem featuredItem) throws InvalidCommandArgumentsException { - if (newDataverseFeaturedItemDTO.getImageFileName() != null) { + private void setImageIfAvailable(DataverseFeaturedItem featuredItem, CommandContext ctxt) throws CommandException { + String imageFileName = newDataverseFeaturedItemDTO.getImageFileName(); + if (imageFileName != null) { try { - prepareUploadedImageFile(); + ctxt.dataverseFeaturedItems().saveDataverseFeaturedItemImageFile(newDataverseFeaturedItemDTO.getFileInputStream(), imageFileName, dataverse.getId()); + } catch (IllegalArgumentException e) { + // TODO check the message that is thrown + throw new InvalidCommandArgumentsException( + e.getMessage(), + this + ); } catch (IOException e) { - throw new RuntimeException(BundleUtil.getStringFromBundle("dataverse.create.featuredItem.error.imageFileProcessing", List.of(e.getMessage())), e); + throw new CommandException( + BundleUtil.getStringFromBundle( + "dataverse.create.featuredItem.error.imageFileProcessing", + List.of(e.getMessage()) + ), + this + ); } - featuredItem.setImageFileName(newDataverseFeaturedItemDTO.getImageFileName()); - } - } - - private void prepareUploadedImageFile() throws IOException, InvalidCommandArgumentsException { - // Step 1: Create a directory to store the uploaded image - Path imageDir = createImageDir(); - File uploadedFile = new File(imageDir.toFile(), newDataverseFeaturedItemDTO.getImageFileName()); - - if (!uploadedFile.exists()) { - uploadedFile.createNewFile(); - } - - // Step 2: Convert the InputStream into a temporary file for validation - File tempFile = FileUtil.inputStreamToFile(newDataverseFeaturedItemDTO.getFileInputStream()); - - // Step 3: Validate the uploaded file (type and size) - validateFile(tempFile); - - // Step 4: Copy the validated file to the final destination - Files.copy(tempFile.toPath(), uploadedFile.toPath(), StandardCopyOption.REPLACE_EXISTING); - } - - private Path createImageDir() throws IOException { - Path imagePath = Path.of(JvmSettings.DOCROOT_DIRECTORY.lookup(), JvmSettings.FEATURED_ITEMS_IMAGE_UPLOADS_DIRECTORY.lookup(), dataverse.getId().toString()); - Files.createDirectories(imagePath); - return imagePath; - } - - private void validateFile(File file) throws IOException, InvalidCommandArgumentsException { - validateFileType(file); - validateFileSize(file); - } - - private void validateFileType(File file) throws IOException, InvalidCommandArgumentsException { - Tika tika = new Tika(); - String mimeType = tika.detect(file); - boolean isImageFile = mimeType != null && mimeType.startsWith("image/"); - if (!isImageFile) { - throw new InvalidCommandArgumentsException( - BundleUtil.getStringFromBundle("dataverse.create.featuredItem.error.invalidFileType"), - this - ); - } - } - - private void validateFileSize(File file) throws InvalidCommandArgumentsException { - Integer featuredItemsImageMaxSize = JvmSettings.FEATURED_ITEMS_IMAGE_MAXSIZE.lookup(Integer.class); - if (file.length() > featuredItemsImageMaxSize) { - throw new InvalidCommandArgumentsException(BundleUtil.getStringFromBundle("dataverse.create.featuredItem.error.fileSizeExceedsLimit", List.of(featuredItemsImageMaxSize.toString())), this); + featuredItem.setImageFileName(imageFileName); } } } diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommandTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommandTest.java new file mode 100644 index 00000000000..8506af85343 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommandTest.java @@ -0,0 +1,120 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.DataverseFeaturedItem; +import edu.harvard.iq.dataverse.DataverseFeaturedItemServiceBean; +import edu.harvard.iq.dataverse.api.dto.NewDataverseFeaturedItemDTO; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import edu.harvard.iq.dataverse.engine.command.exception.InvalidCommandArgumentsException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import java.io.IOException; +import java.io.InputStream; + +import static edu.harvard.iq.dataverse.mocks.MocksFactory.makeRequest; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +class CreateDataverseFeaturedItemCommandTest { + @Mock + private CommandContext contextStub; + + @Mock + private DataverseFeaturedItemServiceBean dataverseFeaturedItemServiceStub; + + @InjectMocks + private CreateDataverseFeaturedItemCommand sut; + + private Dataverse testDataverse; + private NewDataverseFeaturedItemDTO testNewDataverseFeaturedItemDTO; + + @BeforeEach + void setUp() { + MockitoAnnotations.openMocks(this); + + testDataverse = new Dataverse(); + testDataverse.setId(123L); + + testNewDataverseFeaturedItemDTO = new NewDataverseFeaturedItemDTO(); + testNewDataverseFeaturedItemDTO.setImageFileName("test.png"); + testNewDataverseFeaturedItemDTO.setContent("test content"); + testNewDataverseFeaturedItemDTO.setDisplayOrder(0); + testNewDataverseFeaturedItemDTO.setFileInputStream(mock(InputStream.class)); + + when(contextStub.dataverseFeaturedItems()).thenReturn(dataverseFeaturedItemServiceStub); + sut = new CreateDataverseFeaturedItemCommand(makeRequest(), testDataverse, testNewDataverseFeaturedItemDTO); + } + + @Test + void execute_imageFileProvidedAndValid_savesFeaturedItem() throws Exception { + DataverseFeaturedItem expectedFeaturedItem = new DataverseFeaturedItem(); + expectedFeaturedItem.setDataverse(testDataverse); + expectedFeaturedItem.setImageFileName(testNewDataverseFeaturedItemDTO.getImageFileName()); + expectedFeaturedItem.setDisplayOrder(testNewDataverseFeaturedItemDTO.getDisplayOrder()); + expectedFeaturedItem.setContent(testNewDataverseFeaturedItemDTO.getContent()); + + when(dataverseFeaturedItemServiceStub.save(any(DataverseFeaturedItem.class))).thenReturn(expectedFeaturedItem); + + DataverseFeaturedItem result = sut.execute(contextStub); + + assertNotNull(result); + + assertEquals(testNewDataverseFeaturedItemDTO.getImageFileName(), result.getImageFileName()); + assertEquals(testNewDataverseFeaturedItemDTO.getDisplayOrder(), result.getDisplayOrder()); + assertEquals(testNewDataverseFeaturedItemDTO.getContent(), result.getContent()); + assertEquals(testDataverse, result.getDataverse()); + + verify(dataverseFeaturedItemServiceStub).save(any(DataverseFeaturedItem.class)); + verify(dataverseFeaturedItemServiceStub).saveDataverseFeaturedItemImageFile( + testNewDataverseFeaturedItemDTO.getFileInputStream(), + testNewDataverseFeaturedItemDTO.getImageFileName(), + testDataverse.getId() + ); + } + + @Test + void execute_noImageFileProvided_featuredItemSavedWithoutImage() throws Exception { + testNewDataverseFeaturedItemDTO.setImageFileName(null); + + DataverseFeaturedItem expectedFeaturedItem = new DataverseFeaturedItem(); + when(dataverseFeaturedItemServiceStub.save(any(DataverseFeaturedItem.class))).thenReturn(expectedFeaturedItem); + + DataverseFeaturedItem result = sut.execute(contextStub); + + assertNotNull(result); + verify(dataverseFeaturedItemServiceStub).save(any(DataverseFeaturedItem.class)); + verify(dataverseFeaturedItemServiceStub, never()).saveDataverseFeaturedItemImageFile(any(), any(), any()); + } + + @Test + void execute_imageFileProcessingFails_throwsCommandException() throws IOException { + testNewDataverseFeaturedItemDTO.setImageFileName("invalid.png"); + InputStream inputStreamMock = mock(InputStream.class); + testNewDataverseFeaturedItemDTO.setFileInputStream(inputStreamMock); + + doThrow(new IOException("File processing failed")) + .when(dataverseFeaturedItemServiceStub) + .saveDataverseFeaturedItemImageFile(any(InputStream.class), any(String.class), any(Long.class)); + + CommandException exception = assertThrows(CommandException.class, () -> sut.execute(contextStub)); + assertTrue(exception.getMessage().contains("File processing failed")); + } + + @Test + void execute_invalidArgumentsProvided_throwsInvalidCommandArgumentsException() throws IOException { + testNewDataverseFeaturedItemDTO.setImageFileName("invalid.png"); + InputStream inputStreamMock = mock(InputStream.class); + testNewDataverseFeaturedItemDTO.setFileInputStream(inputStreamMock); + + doThrow(new IllegalArgumentException("Invalid file type")) + .when(dataverseFeaturedItemServiceStub).saveDataverseFeaturedItemImageFile(any(InputStream.class), any(String.class), any(Long.class)); + + InvalidCommandArgumentsException exception = assertThrows(InvalidCommandArgumentsException.class, () -> sut.execute(contextStub)); + assertTrue(exception.getMessage().contains("Invalid file type")); + } +} From 64ec37be6387a93ce31e26a455d817b0b5e737ee Mon Sep 17 00:00:00 2001 From: GPortas Date: Tue, 31 Dec 2024 09:41:24 +0000 Subject: [PATCH 059/137] Fixed: exception handling when image file is invalid in a featured item --- .../DataverseFeaturedItemServiceBean.java | 14 ++++++++++---- .../impl/CreateDataverseFeaturedItemCommand.java | 4 ++-- .../CreateDataverseFeaturedItemCommandTest.java | 6 +++--- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java index 1c71f667b5c..acefdeb01a8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java @@ -22,6 +22,12 @@ @Named public class DataverseFeaturedItemServiceBean implements Serializable { + public static class InvalidImageFileException extends Exception { + public InvalidImageFileException(String message) { + super(message); + } + } + @PersistenceContext(unitName = "VDCNet-ejbPU") private EntityManager em; @@ -46,7 +52,7 @@ public InputStream getImageFileAsInputStream(DataverseFeaturedItem dataverseFeat return Files.newInputStream(imagePath); } - public void saveDataverseFeaturedItemImageFile(InputStream inputStream, String imageFileName, Long dataverseId) throws IOException { + public void saveDataverseFeaturedItemImageFile(InputStream inputStream, String imageFileName, Long dataverseId) throws IOException, InvalidImageFileException { File tempFile = FileUtil.inputStreamToFile(inputStream); validateImageFile(tempFile); @@ -68,15 +74,15 @@ private Path createDataverseFeaturedItemImageDir(Long dataverseId) throws IOExce return imagePath; } - private void validateImageFile(File file) throws IOException, IllegalArgumentException { + private void validateImageFile(File file) throws IOException, InvalidImageFileException { if (!isValidDataverseFeaturedItemFileType(file)) { - throw new IllegalArgumentException( + throw new InvalidImageFileException( BundleUtil.getStringFromBundle("dataverse.create.featuredItem.error.invalidFileType") ); } if (!isValidDataverseFeaturedItemFileSize(file)) { String maxAllowedSize = getMaxAllowedDataverseFeaturedItemFileSize().toString(); - throw new IllegalArgumentException( + throw new InvalidImageFileException( BundleUtil.getStringFromBundle("dataverse.create.featuredItem.error.fileSizeExceedsLimit", List.of(maxAllowedSize)) ); } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java index c79f36c10fa..8873a3efea4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java @@ -2,6 +2,7 @@ import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.DataverseFeaturedItem; +import edu.harvard.iq.dataverse.DataverseFeaturedItemServiceBean; import edu.harvard.iq.dataverse.api.dto.NewDataverseFeaturedItemDTO; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.engine.command.AbstractCommand; @@ -45,8 +46,7 @@ private void setImageIfAvailable(DataverseFeaturedItem featuredItem, CommandCont if (imageFileName != null) { try { ctxt.dataverseFeaturedItems().saveDataverseFeaturedItemImageFile(newDataverseFeaturedItemDTO.getFileInputStream(), imageFileName, dataverse.getId()); - } catch (IllegalArgumentException e) { - // TODO check the message that is thrown + } catch (DataverseFeaturedItemServiceBean.InvalidImageFileException e) { throw new InvalidCommandArgumentsException( e.getMessage(), this diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommandTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommandTest.java index 8506af85343..350fc66c230 100644 --- a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommandTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommandTest.java @@ -92,7 +92,7 @@ void execute_noImageFileProvided_featuredItemSavedWithoutImage() throws Exceptio } @Test - void execute_imageFileProcessingFails_throwsCommandException() throws IOException { + void execute_imageFileProcessingFails_throwsCommandException() throws IOException, DataverseFeaturedItemServiceBean.InvalidImageFileException { testNewDataverseFeaturedItemDTO.setImageFileName("invalid.png"); InputStream inputStreamMock = mock(InputStream.class); testNewDataverseFeaturedItemDTO.setFileInputStream(inputStreamMock); @@ -106,12 +106,12 @@ void execute_imageFileProcessingFails_throwsCommandException() throws IOExceptio } @Test - void execute_invalidArgumentsProvided_throwsInvalidCommandArgumentsException() throws IOException { + void execute_invalidArgumentsProvided_throwsInvalidCommandArgumentsException() throws IOException, DataverseFeaturedItemServiceBean.InvalidImageFileException { testNewDataverseFeaturedItemDTO.setImageFileName("invalid.png"); InputStream inputStreamMock = mock(InputStream.class); testNewDataverseFeaturedItemDTO.setFileInputStream(inputStreamMock); - doThrow(new IllegalArgumentException("Invalid file type")) + doThrow(new DataverseFeaturedItemServiceBean.InvalidImageFileException("Invalid file type")) .when(dataverseFeaturedItemServiceStub).saveDataverseFeaturedItemImageFile(any(InputStream.class), any(String.class), any(Long.class)); InvalidCommandArgumentsException exception = assertThrows(InvalidCommandArgumentsException.class, () -> sut.execute(contextStub)); From 98f3645725a54a17dfef2629b4008aab1f2318d9 Mon Sep 17 00:00:00 2001 From: GPortas Date: Tue, 31 Dec 2024 10:47:14 +0000 Subject: [PATCH 060/137] Added: tweaks and fixes for featured items --- .../iq/dataverse/DataverseFeaturedItem.java | 5 ++- .../DataverseFeaturedItemServiceBean.java | 37 +++++-------------- .../edu/harvard/iq/dataverse/api/Access.java | 3 +- .../harvard/iq/dataverse/util/FileUtil.java | 13 +++++++ 4 files changed, 28 insertions(+), 30 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java index ef2421623f3..92bbc5223dd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java @@ -71,6 +71,9 @@ public void setImageFileName(String imageFileName) { } public String getImageFileUrl() { - return SystemConfig.getDataverseSiteUrlStatic() + "/api/access/dataverseFeatureItemImage/" + id; + if (id != null) { + return SystemConfig.getDataverseSiteUrlStatic() + "/api/access/dataverseFeatureItemImage/" + id; + } + return null; } } diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java index acefdeb01a8..1ac5eb76651 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java @@ -7,7 +7,6 @@ import jakarta.inject.Named; import jakarta.persistence.EntityManager; import jakarta.persistence.PersistenceContext; -import org.apache.tika.Tika; import java.io.File; import java.io.IOException; @@ -56,7 +55,11 @@ public void saveDataverseFeaturedItemImageFile(InputStream inputStream, String i File tempFile = FileUtil.inputStreamToFile(inputStream); validateImageFile(tempFile); - Path imageDir = createDataverseFeaturedItemImageDir(dataverseId); + Path imageDir = FileUtil.createDirStructure( + JvmSettings.DOCROOT_DIRECTORY.lookup(), + JvmSettings.FEATURED_ITEMS_IMAGE_UPLOADS_DIRECTORY.lookup(), + dataverseId.toString() + ); File uploadedFile = new File(imageDir.toFile(), imageFileName); if (!uploadedFile.exists()) { @@ -66,39 +69,17 @@ public void saveDataverseFeaturedItemImageFile(InputStream inputStream, String i Files.copy(tempFile.toPath(), uploadedFile.toPath(), StandardCopyOption.REPLACE_EXISTING); } - // TODO: Move file-specific logic out of here - - private Path createDataverseFeaturedItemImageDir(Long dataverseId) throws IOException { - Path imagePath = Path.of(JvmSettings.DOCROOT_DIRECTORY.lookup(), JvmSettings.FEATURED_ITEMS_IMAGE_UPLOADS_DIRECTORY.lookup(), dataverseId.toString()); - Files.createDirectories(imagePath); - return imagePath; - } - private void validateImageFile(File file) throws IOException, InvalidImageFileException { - if (!isValidDataverseFeaturedItemFileType(file)) { + if (!FileUtil.isFileOfImageType(file)) { throw new InvalidImageFileException( BundleUtil.getStringFromBundle("dataverse.create.featuredItem.error.invalidFileType") ); } - if (!isValidDataverseFeaturedItemFileSize(file)) { - String maxAllowedSize = getMaxAllowedDataverseFeaturedItemFileSize().toString(); + Integer maxAllowedSize = JvmSettings.FEATURED_ITEMS_IMAGE_MAXSIZE.lookup(Integer.class); + if (file.length() > maxAllowedSize) { throw new InvalidImageFileException( - BundleUtil.getStringFromBundle("dataverse.create.featuredItem.error.fileSizeExceedsLimit", List.of(maxAllowedSize)) + BundleUtil.getStringFromBundle("dataverse.create.featuredItem.error.fileSizeExceedsLimit", List.of(maxAllowedSize.toString())) ); } } - - private boolean isValidDataverseFeaturedItemFileType(File file) throws IOException { - Tika tika = new Tika(); - String mimeType = tika.detect(file); - return mimeType != null && mimeType.startsWith("image/"); - } - - private boolean isValidDataverseFeaturedItemFileSize(File file) { - return file.length() <= getMaxAllowedDataverseFeaturedItemFileSize(); - } - - private Integer getMaxAllowedDataverseFeaturedItemFileSize() { - return JvmSettings.FEATURED_ITEMS_IMAGE_MAXSIZE.lookup(Integer.class); - } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index d970c5eed44..0bf02411d3a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -1983,9 +1983,10 @@ private URI handleCustomZipDownload(User user, String customZipServiceUrl, Strin return redirectUri; } - @Path("dataverseFeatureItemImage/{itemId}") @GET + @AuthRequired @Produces({"image/png"}) + @Path("dataverseFeatureItemImage/{itemId}") public InputStream getDataverseFeatureItemImage(@Context ContainerRequestContext crc, @PathParam("itemId") Long itemId) { DataverseFeaturedItem dataverseFeaturedItem; try { diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index 991682ec8e8..e134c64277c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -103,6 +103,7 @@ import java.util.Arrays; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.tika.Tika; import ucar.nc2.NetcdfFile; import ucar.nc2.NetcdfFiles; @@ -1828,4 +1829,16 @@ public static String getStorageDriver(DataFile dataFile) { public static String sanitizeFileName(String fileNameIn) { return fileNameIn == null ? null : fileNameIn.replace(' ', '_').replaceAll("[\\\\/:*?\"<>|,;]", ""); } + + public static Path createDirStructure(String rootDirectory, String... subdirectories) throws IOException { + Path path = Path.of(rootDirectory, subdirectories); + Files.createDirectories(path); + return path; + } + + public static boolean isFileOfImageType(File file) throws IOException { + Tika tika = new Tika(); + String mimeType = tika.detect(file); + return mimeType != null && mimeType.startsWith("image/"); + } } From ea03a07df5139153fee6b3ea75d20192dedf01ea Mon Sep 17 00:00:00 2001 From: GPortas Date: Wed, 1 Jan 2025 11:33:30 +0000 Subject: [PATCH 061/137] Added: flush call when saving DataverseFeaturedItem --- .../harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java index 1ac5eb76651..e41d2d59551 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java @@ -37,6 +37,7 @@ public DataverseFeaturedItem findById(Long id) { public DataverseFeaturedItem save(DataverseFeaturedItem dataverseFeaturedItem) { if (dataverseFeaturedItem.getId() == null) { em.persist(dataverseFeaturedItem); + em.flush(); return dataverseFeaturedItem; } else { return em.merge(dataverseFeaturedItem); From ea388ba6a1d665fa4aab11c6994d0abc088b4f23 Mon Sep 17 00:00:00 2001 From: GPortas Date: Wed, 1 Jan 2025 11:33:57 +0000 Subject: [PATCH 062/137] Added: DeleteDataverseFeaturedItemCommand --- .../DeleteDataverseFeaturedItemCommand.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataverseFeaturedItemCommand.java diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataverseFeaturedItemCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataverseFeaturedItemCommand.java new file mode 100644 index 00000000000..7ed0e60cccb --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataverseFeaturedItemCommand.java @@ -0,0 +1,28 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.DataverseFeaturedItem; +import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.engine.command.*; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; + +/** + * Deletes a particular featured item {@link DataverseFeaturedItem} of a {@link Dataverse}. + */ +@RequiredPermissions({Permission.EditDataverse}) +public class DeleteDataverseFeaturedItemCommand extends AbstractVoidCommand { + + private final DataverseFeaturedItem doomed; + + public DeleteDataverseFeaturedItemCommand(DataverseRequest request, DataverseFeaturedItem doomed) { + super(request, doomed.getDataverse()); + this.doomed = doomed; + } + + @Override + protected void executeImpl(CommandContext ctxt) throws CommandException { + ctxt.em().merge(doomed.getDataverse()); + DataverseFeaturedItem doomedAndMerged = ctxt.em().merge(doomed); + ctxt.em().remove(doomedAndMerged); + } +} From 7ede831144f238cd9e3292e4a9b3a96d6e43e01e Mon Sep 17 00:00:00 2001 From: GPortas Date: Wed, 1 Jan 2025 11:34:47 +0000 Subject: [PATCH 063/137] Added: DataverseFeaturedItems API with delete method --- .../dataverse/api/DataverseFeaturedItems.java | 38 +++++++++++++++++++ src/main/java/propertyFiles/Bundle.properties | 4 ++ .../api/DataverseFeaturedItemsIT.java | 27 +++++++++++++ 3 files changed, 69 insertions(+) create mode 100644 src/main/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItems.java create mode 100644 src/test/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItems.java b/src/main/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItems.java new file mode 100644 index 00000000000..de372d55bb1 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItems.java @@ -0,0 +1,38 @@ +package edu.harvard.iq.dataverse.api; + +import edu.harvard.iq.dataverse.*; +import edu.harvard.iq.dataverse.api.auth.AuthRequired; +import edu.harvard.iq.dataverse.engine.command.impl.*; +import edu.harvard.iq.dataverse.util.BundleUtil; +import jakarta.ejb.Stateless; +import jakarta.inject.Inject; +import jakarta.ws.rs.*; +import jakarta.ws.rs.container.ContainerRequestContext; +import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.Response; + +import java.text.MessageFormat; + +@Stateless +@Path("dataverseFeaturedItems") +public class DataverseFeaturedItems extends AbstractApiBean { + + @Inject + DataverseFeaturedItemServiceBean dataverseFeaturedItemServiceBean; + + @DELETE + @AuthRequired + @Path("{itemId}") + public Response deleteItem(@Context ContainerRequestContext crc, @PathParam("itemId") Long itemId) { + try { + DataverseFeaturedItem dataverseFeaturedItem = dataverseFeaturedItemServiceBean.findById(itemId); + if (dataverseFeaturedItem == null) { + throw new WrappedResponse(error(Response.Status.NOT_FOUND, MessageFormat.format(BundleUtil.getStringFromBundle("dataverseFeaturedItems.errors.notFound"), itemId))); + } + execCommand(new DeleteDataverseFeaturedItemCommand(createDataverseRequest(getRequestUser(crc)), dataverseFeaturedItem)); + return ok(MessageFormat.format(BundleUtil.getStringFromBundle("dataverseFeaturedItems.delete.successful"), itemId)); + } catch (WrappedResponse e) { + return e.getResponse(); + } + } +} diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index f01e17dceea..947f914c396 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -3111,3 +3111,7 @@ bearerTokenAuthMechanism.errors.tokenValidatedButNoRegisteredUser=Bearer token i authenticationServiceBean.errors.unauthorizedBearerToken=Unauthorized bearer token. authenticationServiceBean.errors.invalidBearerToken=Could not parse bearer token. authenticationServiceBean.errors.bearerTokenDetectedNoOIDCProviderConfigured=Bearer token detected, no OIDC provider configured. + +#DataverseFeaturedItems.java +dataverseFeaturedItems.errors.notFound=Can't find dataverse featured item with identifier='{0}' +dataverseFeaturedItems.delete.successful=Successfully deleted dataverse featured item with identifier='{0}' diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java new file mode 100644 index 00000000000..d3e6a3f9364 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java @@ -0,0 +1,27 @@ +package edu.harvard.iq.dataverse.api; + +import io.restassured.RestAssured; +import io.restassured.response.Response; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import static jakarta.ws.rs.core.Response.Status.*; + +public class DataverseFeaturedItemsIT { + + @BeforeAll + public static void setUpClass() { + RestAssured.baseURI = UtilIT.getRestAssuredBaseUri(); + } + + @Test + public void testDeleteFeaturedItem() { + Response createUserResponse = UtilIT.createRandomUser(); + String apiToken = UtilIT.getApiTokenFromResponse(createUserResponse); + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode()); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + // TODO + } +} From d50a943c7e65172470d310743570e849bba31396 Mon Sep 17 00:00:00 2001 From: GPortas Date: Wed, 1 Jan 2025 14:00:26 +0000 Subject: [PATCH 064/137] Added: DataverseFeaturedItems API with delete method IT --- .../iq/dataverse/DataverseFeaturedItem.java | 3 +-- .../dataverse/api/DataverseFeaturedItems.java | 10 ++++----- .../api/DataverseFeaturedItemsIT.java | 22 ++++++++++++++++++- .../edu/harvard/iq/dataverse/api/UtilIT.java | 8 ++++++- tests/integration-tests.txt | 2 +- 5 files changed, 35 insertions(+), 10 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java index 92bbc5223dd..d0040745bdf 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java @@ -7,7 +7,6 @@ import jakarta.validation.constraints.Size; @Entity -@Table(indexes = {@Index(columnList = "dataverse_id"), @Index(columnList = "displayOrder")}) public class DataverseFeaturedItem { @Id @@ -15,7 +14,7 @@ public class DataverseFeaturedItem { private Long id; @ManyToOne - @JoinColumn(name = "dataverse_id", nullable = false) + @JoinColumn(nullable = false) private Dataverse dataverse; @NotBlank diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItems.java b/src/main/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItems.java index de372d55bb1..4d4c246a6b1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItems.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItems.java @@ -22,15 +22,15 @@ public class DataverseFeaturedItems extends AbstractApiBean { @DELETE @AuthRequired - @Path("{itemId}") - public Response deleteItem(@Context ContainerRequestContext crc, @PathParam("itemId") Long itemId) { + @Path("{id}") + public Response deleteFeaturedItem(@Context ContainerRequestContext crc, @PathParam("id") Long id) { try { - DataverseFeaturedItem dataverseFeaturedItem = dataverseFeaturedItemServiceBean.findById(itemId); + DataverseFeaturedItem dataverseFeaturedItem = dataverseFeaturedItemServiceBean.findById(id); if (dataverseFeaturedItem == null) { - throw new WrappedResponse(error(Response.Status.NOT_FOUND, MessageFormat.format(BundleUtil.getStringFromBundle("dataverseFeaturedItems.errors.notFound"), itemId))); + throw new WrappedResponse(error(Response.Status.NOT_FOUND, MessageFormat.format(BundleUtil.getStringFromBundle("dataverseFeaturedItems.errors.notFound"), id))); } execCommand(new DeleteDataverseFeaturedItemCommand(createDataverseRequest(getRequestUser(crc)), dataverseFeaturedItem)); - return ok(MessageFormat.format(BundleUtil.getStringFromBundle("dataverseFeaturedItems.delete.successful"), itemId)); + return ok(MessageFormat.format(BundleUtil.getStringFromBundle("dataverseFeaturedItems.delete.successful"), id)); } catch (WrappedResponse e) { return e.getResponse(); } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java index d3e6a3f9364..57d21ceb4f9 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java @@ -1,6 +1,7 @@ package edu.harvard.iq.dataverse.api; import io.restassured.RestAssured; +import io.restassured.path.json.JsonPath; import io.restassured.response.Response; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -22,6 +23,25 @@ public void testDeleteFeaturedItem() { createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode()); String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); - // TODO + String pathToTestFile = "src/test/resources/images/coffeeshop.png"; + Response createFeatureItemResponse = UtilIT.createDataverseFeaturedItem(dataverseAlias, apiToken, "test", "test", pathToTestFile); + createFeatureItemResponse.then().assertThat().statusCode(OK.getStatusCode()); + + JsonPath createdFeaturedItem = JsonPath.from(createFeatureItemResponse.body().asString()); + Long featuredItemId = createdFeaturedItem.getLong("data.id"); + + // Should return not found when passing incorrect item id + Response deleteFeatureItemResponse = UtilIT.deleteDataverseFeaturedItem(100000L, apiToken); + deleteFeatureItemResponse.then().assertThat().statusCode(NOT_FOUND.getStatusCode()); + + // Should return unauthorized when passing correct id and user does not have permissions + Response createRandomUser = UtilIT.createRandomUser(); + String randomUserApiToken = UtilIT.getApiTokenFromResponse(createRandomUser); + deleteFeatureItemResponse = UtilIT.deleteDataverseFeaturedItem(featuredItemId, randomUserApiToken); + deleteFeatureItemResponse.then().assertThat().statusCode(UNAUTHORIZED.getStatusCode()); + + // Should delete featured item when passing correct id and user have permissions + deleteFeatureItemResponse = UtilIT.deleteDataverseFeaturedItem(featuredItemId, apiToken); + deleteFeatureItemResponse.then().assertThat().statusCode(OK.getStatusCode()); } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 6c4eae0d06f..11d303156ae 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -4360,7 +4360,7 @@ static Response performKeycloakROPCLogin(String username, String password) { .post("http://keycloak.mydomain.com:8090/realms/test/protocol/openid-connect/token"); } - static Response createFeaturedItem(String dataverseAlias, String apiToken, String title, String content, String pathToFile) { + static Response createDataverseFeaturedItem(String dataverseAlias, String apiToken, String title, String content, String pathToFile) { return given() .header(API_TOKEN_HTTP_HEADER, apiToken) .contentType(ContentType.MULTIPART) @@ -4371,6 +4371,12 @@ static Response createFeaturedItem(String dataverseAlias, String apiToken, Strin .post("/api/dataverses/" + dataverseAlias + "/featuredItem"); } + static Response deleteDataverseFeaturedItem(long id, String apiToken) { + return given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .delete("/api/dataverseFeaturedItems/" + id); + } + static Response listDataverseFeaturedItems(String dataverseAlias, String apiToken) { return given() .header(API_TOKEN_HTTP_HEADER, apiToken) diff --git a/tests/integration-tests.txt b/tests/integration-tests.txt index e1dad7a75b1..a20d65e26e3 100644 --- a/tests/integration-tests.txt +++ b/tests/integration-tests.txt @@ -1 +1 @@ -DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,HarvestingClientsIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT,DeleteUsersIT,DeactivateUsersIT,AuxiliaryFilesIT,InvalidCharactersIT,LicensesIT,NotificationsIT,BagIT,MetadataBlocksIT,NetcdfIT,SignpostingIT,FitsIT,LogoutIT,DataRetrieverApiIT,ProvIT,S3AccessIT,OpenApiIT,InfoIT,DatasetFieldsIT,SavedSearchIT,DatasetTypesIT \ No newline at end of file +DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,HarvestingClientsIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT,DeleteUsersIT,DeactivateUsersIT,AuxiliaryFilesIT,InvalidCharactersIT,LicensesIT,NotificationsIT,BagIT,MetadataBlocksIT,NetcdfIT,SignpostingIT,FitsIT,LogoutIT,DataRetrieverApiIT,ProvIT,S3AccessIT,OpenApiIT,InfoIT,DatasetFieldsIT,SavedSearchIT,DatasetTypesIT,DataverseFeaturedItemsIT \ No newline at end of file From 220e02a1749bc9d7af617cc7a993768529fa1b4c Mon Sep 17 00:00:00 2001 From: GPortas Date: Wed, 1 Jan 2025 14:27:07 +0000 Subject: [PATCH 065/137] Added: DataverseFeaturedItemsIT assertion --- .../harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java index 57d21ceb4f9..58b33183e54 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java @@ -7,6 +7,7 @@ import org.junit.jupiter.api.Test; import static jakarta.ws.rs.core.Response.Status.*; +import static org.hamcrest.CoreMatchers.equalTo; public class DataverseFeaturedItemsIT { @@ -43,5 +44,10 @@ public void testDeleteFeaturedItem() { // Should delete featured item when passing correct id and user have permissions deleteFeatureItemResponse = UtilIT.deleteDataverseFeaturedItem(featuredItemId, apiToken); deleteFeatureItemResponse.then().assertThat().statusCode(OK.getStatusCode()); + + Response listFeaturedItemsResponse = UtilIT.listDataverseFeaturedItems(dataverseAlias, apiToken); + listFeaturedItemsResponse.then() + .body("data.size()", equalTo(0)) + .assertThat().statusCode(OK.getStatusCode()); } } From 1bc2ba74f413e617972ef4e894d2fa934941791e Mon Sep 17 00:00:00 2001 From: GPortas Date: Wed, 1 Jan 2025 14:37:03 +0000 Subject: [PATCH 066/137] Added: testCreateFeaturedItem IT cases --- .../iq/dataverse/api/DataversesIT.java | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 488e2d93120..a083716d2e2 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -1587,22 +1587,30 @@ public void testCreateFeaturedItem() { // Should not return any error when passing correct file and data String pathToTestFile = "src/test/resources/images/coffeeshop.png"; - Response updateFeatureItemsResponse = UtilIT.createFeaturedItem(dataverseAlias, apiToken, "test", "test", pathToTestFile); - updateFeatureItemsResponse.then().assertThat() + Response createFeatureItemResponse = UtilIT.createDataverseFeaturedItem(dataverseAlias, apiToken, "test", "test", pathToTestFile); + createFeatureItemResponse.then().assertThat() .body("data.content", equalTo("test")) .body("data.imageFileName", equalTo("coffeeshop.png")) .body("data.displayOrder", equalTo(0)) .statusCode(OK.getStatusCode()); - // TODO - Response listFeaturedItemsResponse = UtilIT.listDataverseFeaturedItems(dataverseAlias, apiToken); - listFeaturedItemsResponse.prettyPrint(); - - // Should return error when passing incorrect file type - pathToTestFile = "src/test/resources/tab/test.tab"; - updateFeatureItemsResponse = UtilIT.createFeaturedItem(dataverseAlias, apiToken, "test", "test", pathToTestFile); - updateFeatureItemsResponse.then().assertThat() + // Should return bad request error when passing incorrect file type + pathToTestFile = "src/test/resources/tab/test.tab"; + createFeatureItemResponse = UtilIT.createDataverseFeaturedItem(dataverseAlias, apiToken, "test", "test", pathToTestFile); + createFeatureItemResponse.then().assertThat() .body("message", equalTo(BundleUtil.getStringFromBundle("dataverse.create.featuredItem.error.invalidFileType"))) .statusCode(BAD_REQUEST.getStatusCode()); + + // Should return unauthorized error when user has no permissions + Response createRandomUser = UtilIT.createRandomUser(); + String randomUserApiToken = UtilIT.getApiTokenFromResponse(createRandomUser); + createFeatureItemResponse = UtilIT.createDataverseFeaturedItem(dataverseAlias, randomUserApiToken, "test", "test", pathToTestFile); + createFeatureItemResponse.then().assertThat().statusCode(UNAUTHORIZED.getStatusCode()); + + // Should return not found error when dataverse does not exist + createFeatureItemResponse = UtilIT.createDataverseFeaturedItem("thisDataverseDoesNotExist", apiToken, "test", "test", pathToTestFile); + createFeatureItemResponse.then().assertThat() + .body("message", equalTo("Can't find dataverse with identifier='thisDataverseDoesNotExist'")) + .statusCode(NOT_FOUND.getStatusCode()); } } From f7d60c7269eb4391d8944be8656a84fe3e0aac37 Mon Sep 17 00:00:00 2001 From: GPortas Date: Thu, 2 Jan 2025 14:06:37 +0000 Subject: [PATCH 067/137] Added: handling image file optionality in dataverse featured items --- .../iq/dataverse/DataverseFeaturedItem.java | 2 +- .../harvard/iq/dataverse/api/Dataverses.java | 6 ++--- .../api/dto/NewDataverseFeaturedItemDTO.java | 25 +++++++++++-------- .../CreateDataverseFeaturedItemCommand.java | 6 +++-- .../api/DataverseFeaturedItemsIT.java | 2 +- .../iq/dataverse/api/DataversesIT.java | 18 +++++++++---- .../edu/harvard/iq/dataverse/api/UtilIT.java | 12 ++++++--- ...reateDataverseFeaturedItemCommandTest.java | 8 +++--- 8 files changed, 50 insertions(+), 29 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java index d0040745bdf..52da2bb9be5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java @@ -70,7 +70,7 @@ public void setImageFileName(String imageFileName) { } public String getImageFileUrl() { - if (id != null) { + if (id != null && imageFileName != null) { return SystemConfig.getDataverseSiteUrlStatic() + "/api/access/dataverseFeatureItemImage/" + id; } return null; diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index b2f174d9449..015485c9a9e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -1738,8 +1738,8 @@ public Response getUserPermissionsOnDataverse(@Context ContainerRequestContext c public Response createFeaturedItem(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf, @FormDataParam("content") String content, - @FormDataParam("order") int order, - @FormDataParam("file") InputStream fileInputStream, + @FormDataParam("displayOrder") int displayOrder, + @FormDataParam("file") InputStream imageFileInputStream, @FormDataParam("file") FormDataContentDisposition contentDispositionHeader) { Dataverse dataverse; try { @@ -1747,7 +1747,7 @@ public Response createFeaturedItem(@Context ContainerRequestContext crc, } catch (WrappedResponse wr) { return wr.getResponse(); } - NewDataverseFeaturedItemDTO newDataverseFeaturedItemDTO = NewDataverseFeaturedItemDTO.fromFormData(content, order, fileInputStream, contentDispositionHeader); + NewDataverseFeaturedItemDTO newDataverseFeaturedItemDTO = NewDataverseFeaturedItemDTO.fromFormData(content, displayOrder, imageFileInputStream, contentDispositionHeader); try { DataverseFeaturedItem dataverseFeaturedItem = execCommand(new CreateDataverseFeaturedItemCommand( createDataverseRequest(getRequestUser(crc)), diff --git a/src/main/java/edu/harvard/iq/dataverse/api/dto/NewDataverseFeaturedItemDTO.java b/src/main/java/edu/harvard/iq/dataverse/api/dto/NewDataverseFeaturedItemDTO.java index e1f5190cb41..47003761abc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/dto/NewDataverseFeaturedItemDTO.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/dto/NewDataverseFeaturedItemDTO.java @@ -7,18 +7,23 @@ public class NewDataverseFeaturedItemDTO { private String content; private int displayOrder; - private InputStream fileInputStream; + private InputStream imageFileInputStream; private String imageFileName; public static NewDataverseFeaturedItemDTO fromFormData(String content, - int order, - InputStream fileInputStream, + int displayOrder, + InputStream imageFileInputStream, FormDataContentDisposition contentDispositionHeader) { NewDataverseFeaturedItemDTO newDataverseFeaturedItemDTO = new NewDataverseFeaturedItemDTO(); + newDataverseFeaturedItemDTO.content = content; - newDataverseFeaturedItemDTO.displayOrder = order; - newDataverseFeaturedItemDTO.fileInputStream = fileInputStream; - newDataverseFeaturedItemDTO.imageFileName = contentDispositionHeader.getFileName(); + newDataverseFeaturedItemDTO.displayOrder = displayOrder; + + if (imageFileInputStream != null) { + newDataverseFeaturedItemDTO.imageFileInputStream = imageFileInputStream; + newDataverseFeaturedItemDTO.imageFileName = contentDispositionHeader.getFileName(); + } + return newDataverseFeaturedItemDTO; } @@ -38,12 +43,12 @@ public int getDisplayOrder() { return displayOrder; } - public void setFileInputStream(InputStream fileInputStream) { - this.fileInputStream = fileInputStream; + public void setImageFileInputStream(InputStream imageFileInputStream) { + this.imageFileInputStream = imageFileInputStream; } - public InputStream getFileInputStream() { - return fileInputStream; + public InputStream getImageFileInputStream() { + return imageFileInputStream; } public void setImageFileName(String imageFileName) { diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java index 8873a3efea4..1bd296fb384 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java @@ -14,6 +14,7 @@ import edu.harvard.iq.dataverse.util.BundleUtil; import java.io.IOException; +import java.io.InputStream; import java.util.List; @RequiredPermissions({Permission.EditDataverse}) @@ -43,9 +44,10 @@ public DataverseFeaturedItem execute(CommandContext ctxt) throws CommandExceptio private void setImageIfAvailable(DataverseFeaturedItem featuredItem, CommandContext ctxt) throws CommandException { String imageFileName = newDataverseFeaturedItemDTO.getImageFileName(); - if (imageFileName != null) { + InputStream imageFileInputStream = newDataverseFeaturedItemDTO.getImageFileInputStream(); + if (imageFileName != null && imageFileInputStream != null) { try { - ctxt.dataverseFeaturedItems().saveDataverseFeaturedItemImageFile(newDataverseFeaturedItemDTO.getFileInputStream(), imageFileName, dataverse.getId()); + ctxt.dataverseFeaturedItems().saveDataverseFeaturedItemImageFile(imageFileInputStream, imageFileName, dataverse.getId()); } catch (DataverseFeaturedItemServiceBean.InvalidImageFileException e) { throw new InvalidCommandArgumentsException( e.getMessage(), diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java index 58b33183e54..765b1cb25ae 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java @@ -25,7 +25,7 @@ public void testDeleteFeaturedItem() { String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); String pathToTestFile = "src/test/resources/images/coffeeshop.png"; - Response createFeatureItemResponse = UtilIT.createDataverseFeaturedItem(dataverseAlias, apiToken, "test", "test", pathToTestFile); + Response createFeatureItemResponse = UtilIT.createDataverseFeaturedItem(dataverseAlias, apiToken, "test", "test", 0, pathToTestFile); createFeatureItemResponse.then().assertThat().statusCode(OK.getStatusCode()); JsonPath createdFeaturedItem = JsonPath.from(createFeatureItemResponse.body().asString()); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index a083716d2e2..640e0d42245 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -1585,18 +1585,26 @@ public void testCreateFeaturedItem() { createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode()); String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + // Should not return any error when not passing a file + Response createFeatureItemResponse = UtilIT.createDataverseFeaturedItem(dataverseAlias, apiToken, "test", "test", 0, null); + createFeatureItemResponse.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.content", equalTo("test")) + .body("data.imageFileName", equalTo(null)) + .body("data.displayOrder", equalTo(0)); + // Should not return any error when passing correct file and data String pathToTestFile = "src/test/resources/images/coffeeshop.png"; - Response createFeatureItemResponse = UtilIT.createDataverseFeaturedItem(dataverseAlias, apiToken, "test", "test", pathToTestFile); + createFeatureItemResponse = UtilIT.createDataverseFeaturedItem(dataverseAlias, apiToken, "test", "test", 1, pathToTestFile); createFeatureItemResponse.then().assertThat() .body("data.content", equalTo("test")) .body("data.imageFileName", equalTo("coffeeshop.png")) - .body("data.displayOrder", equalTo(0)) + .body("data.displayOrder", equalTo(1)) .statusCode(OK.getStatusCode()); // Should return bad request error when passing incorrect file type pathToTestFile = "src/test/resources/tab/test.tab"; - createFeatureItemResponse = UtilIT.createDataverseFeaturedItem(dataverseAlias, apiToken, "test", "test", pathToTestFile); + createFeatureItemResponse = UtilIT.createDataverseFeaturedItem(dataverseAlias, apiToken, "test", "test", 0, pathToTestFile); createFeatureItemResponse.then().assertThat() .body("message", equalTo(BundleUtil.getStringFromBundle("dataverse.create.featuredItem.error.invalidFileType"))) .statusCode(BAD_REQUEST.getStatusCode()); @@ -1604,11 +1612,11 @@ public void testCreateFeaturedItem() { // Should return unauthorized error when user has no permissions Response createRandomUser = UtilIT.createRandomUser(); String randomUserApiToken = UtilIT.getApiTokenFromResponse(createRandomUser); - createFeatureItemResponse = UtilIT.createDataverseFeaturedItem(dataverseAlias, randomUserApiToken, "test", "test", pathToTestFile); + createFeatureItemResponse = UtilIT.createDataverseFeaturedItem(dataverseAlias, randomUserApiToken, "test", "test", 0, pathToTestFile); createFeatureItemResponse.then().assertThat().statusCode(UNAUTHORIZED.getStatusCode()); // Should return not found error when dataverse does not exist - createFeatureItemResponse = UtilIT.createDataverseFeaturedItem("thisDataverseDoesNotExist", apiToken, "test", "test", pathToTestFile); + createFeatureItemResponse = UtilIT.createDataverseFeaturedItem("thisDataverseDoesNotExist", apiToken, "test", "test", 0, pathToTestFile); createFeatureItemResponse.then().assertThat() .body("message", equalTo("Can't find dataverse with identifier='thisDataverseDoesNotExist'")) .statusCode(NOT_FOUND.getStatusCode()); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 11d303156ae..9591f84bdf6 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -4360,13 +4360,19 @@ static Response performKeycloakROPCLogin(String username, String password) { .post("http://keycloak.mydomain.com:8090/realms/test/protocol/openid-connect/token"); } - static Response createDataverseFeaturedItem(String dataverseAlias, String apiToken, String title, String content, String pathToFile) { - return given() + static Response createDataverseFeaturedItem(String dataverseAlias, String apiToken, String title, String content, int displayOrder, String pathToFile) { + RequestSpecification requestSpecification = given() .header(API_TOKEN_HTTP_HEADER, apiToken) .contentType(ContentType.MULTIPART) .multiPart("title", title) .multiPart("content", content) - .multiPart("file", new File(pathToFile)) + .multiPart("displayOrder", displayOrder); + + if (pathToFile != null) { + requestSpecification.multiPart("file", new File(pathToFile)); + } + + return requestSpecification .when() .post("/api/dataverses/" + dataverseAlias + "/featuredItem"); } diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommandTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommandTest.java index 350fc66c230..d8bb6549c60 100644 --- a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommandTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommandTest.java @@ -44,7 +44,7 @@ void setUp() { testNewDataverseFeaturedItemDTO.setImageFileName("test.png"); testNewDataverseFeaturedItemDTO.setContent("test content"); testNewDataverseFeaturedItemDTO.setDisplayOrder(0); - testNewDataverseFeaturedItemDTO.setFileInputStream(mock(InputStream.class)); + testNewDataverseFeaturedItemDTO.setImageFileInputStream(mock(InputStream.class)); when(contextStub.dataverseFeaturedItems()).thenReturn(dataverseFeaturedItemServiceStub); sut = new CreateDataverseFeaturedItemCommand(makeRequest(), testDataverse, testNewDataverseFeaturedItemDTO); @@ -71,7 +71,7 @@ void execute_imageFileProvidedAndValid_savesFeaturedItem() throws Exception { verify(dataverseFeaturedItemServiceStub).save(any(DataverseFeaturedItem.class)); verify(dataverseFeaturedItemServiceStub).saveDataverseFeaturedItemImageFile( - testNewDataverseFeaturedItemDTO.getFileInputStream(), + testNewDataverseFeaturedItemDTO.getImageFileInputStream(), testNewDataverseFeaturedItemDTO.getImageFileName(), testDataverse.getId() ); @@ -95,7 +95,7 @@ void execute_noImageFileProvided_featuredItemSavedWithoutImage() throws Exceptio void execute_imageFileProcessingFails_throwsCommandException() throws IOException, DataverseFeaturedItemServiceBean.InvalidImageFileException { testNewDataverseFeaturedItemDTO.setImageFileName("invalid.png"); InputStream inputStreamMock = mock(InputStream.class); - testNewDataverseFeaturedItemDTO.setFileInputStream(inputStreamMock); + testNewDataverseFeaturedItemDTO.setImageFileInputStream(inputStreamMock); doThrow(new IOException("File processing failed")) .when(dataverseFeaturedItemServiceStub) @@ -109,7 +109,7 @@ void execute_imageFileProcessingFails_throwsCommandException() throws IOExceptio void execute_invalidArgumentsProvided_throwsInvalidCommandArgumentsException() throws IOException, DataverseFeaturedItemServiceBean.InvalidImageFileException { testNewDataverseFeaturedItemDTO.setImageFileName("invalid.png"); InputStream inputStreamMock = mock(InputStream.class); - testNewDataverseFeaturedItemDTO.setFileInputStream(inputStreamMock); + testNewDataverseFeaturedItemDTO.setImageFileInputStream(inputStreamMock); doThrow(new DataverseFeaturedItemServiceBean.InvalidImageFileException("Invalid file type")) .when(dataverseFeaturedItemServiceStub).saveDataverseFeaturedItemImageFile(any(InputStream.class), any(String.class), any(Long.class)); From 034cea463a520c1f875f05c920d0c38aa3da55a4 Mon Sep 17 00:00:00 2001 From: GPortas Date: Thu, 2 Jan 2025 15:34:43 +0000 Subject: [PATCH 068/137] Added: API endpoint for updating a dataverse featured item --- .../DataverseFeaturedItemServiceBean.java | 6 +- .../dataverse/api/DataverseFeaturedItems.java | 30 ++++++++ .../dto/UpdatedDataverseFeaturedItemDTO.java | 72 ++++++++++++++++++ ...ractWriteDataverseFeaturedItemCommand.java | 53 +++++++++++++ .../CreateDataverseFeaturedItemCommand.java | 60 ++++----------- .../UpdateDataverseFeaturedItemCommand.java | 41 ++++++++++ .../api/DataverseFeaturedItemsIT.java | 74 +++++++++++++++---- .../iq/dataverse/api/DataversesIT.java | 10 +-- .../edu/harvard/iq/dataverse/api/UtilIT.java | 29 +++++++- 9 files changed, 305 insertions(+), 70 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/api/dto/UpdatedDataverseFeaturedItemDTO.java create mode 100644 src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseFeaturedItemCommand.java create mode 100644 src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemCommand.java diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java index e41d2d59551..90fb3e04d1f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java @@ -37,11 +37,11 @@ public DataverseFeaturedItem findById(Long id) { public DataverseFeaturedItem save(DataverseFeaturedItem dataverseFeaturedItem) { if (dataverseFeaturedItem.getId() == null) { em.persist(dataverseFeaturedItem); - em.flush(); - return dataverseFeaturedItem; } else { - return em.merge(dataverseFeaturedItem); + dataverseFeaturedItem = em.merge(dataverseFeaturedItem); } + em.flush(); + return dataverseFeaturedItem; } public InputStream getImageFileAsInputStream(DataverseFeaturedItem dataverseFeaturedItem) throws IOException { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItems.java b/src/main/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItems.java index 4d4c246a6b1..fdaccaa08d2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItems.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItems.java @@ -2,6 +2,7 @@ import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.api.auth.AuthRequired; +import edu.harvard.iq.dataverse.api.dto.UpdatedDataverseFeaturedItemDTO; import edu.harvard.iq.dataverse.engine.command.impl.*; import edu.harvard.iq.dataverse.util.BundleUtil; import jakarta.ejb.Stateless; @@ -9,10 +10,16 @@ import jakarta.ws.rs.*; import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; +import org.glassfish.jersey.media.multipart.FormDataContentDisposition; +import org.glassfish.jersey.media.multipart.FormDataParam; +import java.io.InputStream; import java.text.MessageFormat; +import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json; + @Stateless @Path("dataverseFeaturedItems") public class DataverseFeaturedItems extends AbstractApiBean { @@ -35,4 +42,27 @@ public Response deleteFeaturedItem(@Context ContainerRequestContext crc, @PathPa return e.getResponse(); } } + + @PUT + @AuthRequired + @Consumes(MediaType.MULTIPART_FORM_DATA) + @Path("{id}") + public Response updateFeaturedItem(@Context ContainerRequestContext crc, + @PathParam("id") Long id, + @FormDataParam("content") String content, + @FormDataParam("displayOrder") int displayOrder, + @FormDataParam("keepFile") boolean keepFile, + @FormDataParam("file") InputStream imageFileInputStream, + @FormDataParam("file") FormDataContentDisposition contentDispositionHeader) { + try { + DataverseFeaturedItem dataverseFeaturedItem = dataverseFeaturedItemServiceBean.findById(id); + if (dataverseFeaturedItem == null) { + throw new WrappedResponse(error(Response.Status.NOT_FOUND, MessageFormat.format(BundleUtil.getStringFromBundle("dataverseFeaturedItems.errors.notFound"), id))); + } + UpdatedDataverseFeaturedItemDTO updatedDataverseFeaturedItemDTO = UpdatedDataverseFeaturedItemDTO.fromFormData(content, displayOrder, keepFile, imageFileInputStream, contentDispositionHeader); + return ok(json(execCommand(new UpdateDataverseFeaturedItemCommand(createDataverseRequest(getRequestUser(crc)), dataverseFeaturedItem, updatedDataverseFeaturedItemDTO)))); + } catch (WrappedResponse e) { + return e.getResponse(); + } + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/dto/UpdatedDataverseFeaturedItemDTO.java b/src/main/java/edu/harvard/iq/dataverse/api/dto/UpdatedDataverseFeaturedItemDTO.java new file mode 100644 index 00000000000..43d1afc31e2 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/api/dto/UpdatedDataverseFeaturedItemDTO.java @@ -0,0 +1,72 @@ +package edu.harvard.iq.dataverse.api.dto; + +import org.glassfish.jersey.media.multipart.FormDataContentDisposition; + +import java.io.InputStream; + +public class UpdatedDataverseFeaturedItemDTO { + private String content; + private int displayOrder; + private boolean keepFile; + private InputStream imageFileInputStream; + private String imageFileName; + + public static UpdatedDataverseFeaturedItemDTO fromFormData(String content, + int displayOrder, + boolean keepFile, + InputStream imageFileInputStream, + FormDataContentDisposition contentDispositionHeader) { + UpdatedDataverseFeaturedItemDTO updatedDataverseFeaturedItemDTO = new UpdatedDataverseFeaturedItemDTO(); + + updatedDataverseFeaturedItemDTO.content = content; + updatedDataverseFeaturedItemDTO.displayOrder = displayOrder; + updatedDataverseFeaturedItemDTO.keepFile = keepFile; + + if (imageFileInputStream != null) { + updatedDataverseFeaturedItemDTO.imageFileInputStream = imageFileInputStream; + updatedDataverseFeaturedItemDTO.imageFileName = contentDispositionHeader.getFileName(); + } + + return updatedDataverseFeaturedItemDTO; + } + + public void setContent(String content) { + this.content = content; + } + + public String getContent() { + return content; + } + + public void setDisplayOrder(int displayOrder) { + this.displayOrder = displayOrder; + } + + public int getDisplayOrder() { + return displayOrder; + } + + public void setKeepFile(boolean keepFile) { + this.keepFile = keepFile; + } + + public boolean isKeepFile() { + return keepFile; + } + + public void setImageFileInputStream(InputStream imageFileInputStream) { + this.imageFileInputStream = imageFileInputStream; + } + + public InputStream getImageFileInputStream() { + return imageFileInputStream; + } + + public void setImageFileName(String imageFileName) { + this.imageFileName = imageFileName; + } + + public String getImageFileName() { + return imageFileName; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseFeaturedItemCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseFeaturedItemCommand.java new file mode 100644 index 00000000000..0b3395d6f78 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseFeaturedItemCommand.java @@ -0,0 +1,53 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.*; +import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.engine.command.AbstractCommand; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import edu.harvard.iq.dataverse.engine.command.exception.InvalidCommandArgumentsException; +import edu.harvard.iq.dataverse.util.BundleUtil; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + +/** + * An abstract base class for commands that perform write operations on {@link DataverseFeaturedItem}s. + */ +@RequiredPermissions({Permission.EditDataverse}) +abstract class AbstractWriteDataverseFeaturedItemCommand extends AbstractCommand { + + protected final Dataverse dataverse; + + public AbstractWriteDataverseFeaturedItemCommand(DataverseRequest request, Dataverse affectedDataverse) { + super(request, affectedDataverse); + this.dataverse = affectedDataverse; + } + + protected void setFileImageIfAvailableOrNull(DataverseFeaturedItem featuredItem, String imageFileName, InputStream imageFileInputStream, CommandContext ctxt) throws CommandException { + if (imageFileName != null && imageFileInputStream != null) { + try { + ctxt.dataverseFeaturedItems().saveDataverseFeaturedItemImageFile(imageFileInputStream, imageFileName, dataverse.getId()); + } catch (DataverseFeaturedItemServiceBean.InvalidImageFileException e) { + throw new InvalidCommandArgumentsException( + e.getMessage(), + this + ); + } catch (IOException e) { + throw new CommandException( + BundleUtil.getStringFromBundle( + "dataverse.create.featuredItem.error.imageFileProcessing", + List.of(e.getMessage()) + ), + this + ); + } + featuredItem.setImageFileName(imageFileName); + } else { + featuredItem.setImageFileName(null); + } + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java index 1bd296fb384..e1ede59817c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java @@ -2,67 +2,37 @@ import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.DataverseFeaturedItem; -import edu.harvard.iq.dataverse.DataverseFeaturedItemServiceBean; import edu.harvard.iq.dataverse.api.dto.NewDataverseFeaturedItemDTO; -import edu.harvard.iq.dataverse.authorization.Permission; -import edu.harvard.iq.dataverse.engine.command.AbstractCommand; import edu.harvard.iq.dataverse.engine.command.CommandContext; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; -import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; -import edu.harvard.iq.dataverse.engine.command.exception.InvalidCommandArgumentsException; -import edu.harvard.iq.dataverse.util.BundleUtil; -import java.io.IOException; -import java.io.InputStream; -import java.util.List; +public class CreateDataverseFeaturedItemCommand extends AbstractWriteDataverseFeaturedItemCommand { -@RequiredPermissions({Permission.EditDataverse}) -public class CreateDataverseFeaturedItemCommand extends AbstractCommand { - - private final Dataverse dataverse; private final NewDataverseFeaturedItemDTO newDataverseFeaturedItemDTO; - public CreateDataverseFeaturedItemCommand(DataverseRequest request, Dataverse dataverse, NewDataverseFeaturedItemDTO newDataverseFeaturedItemDTO) { + public CreateDataverseFeaturedItemCommand(DataverseRequest request, + Dataverse dataverse, + NewDataverseFeaturedItemDTO newDataverseFeaturedItemDTO) { super(request, dataverse); - this.dataverse = dataverse; this.newDataverseFeaturedItemDTO = newDataverseFeaturedItemDTO; } @Override public DataverseFeaturedItem execute(CommandContext ctxt) throws CommandException { - DataverseFeaturedItem featuredItem = new DataverseFeaturedItem(); - - setImageIfAvailable(featuredItem, ctxt); + DataverseFeaturedItem dataverseFeaturedItem = new DataverseFeaturedItem(); - featuredItem.setContent(newDataverseFeaturedItemDTO.getContent()); - featuredItem.setDisplayOrder(newDataverseFeaturedItemDTO.getDisplayOrder()); - featuredItem.setDataverse(dataverse); + setFileImageIfAvailableOrNull( + dataverseFeaturedItem, + newDataverseFeaturedItemDTO.getImageFileName(), + newDataverseFeaturedItemDTO.getImageFileInputStream(), + ctxt + ); - return ctxt.dataverseFeaturedItems().save(featuredItem); - } + dataverseFeaturedItem.setContent(newDataverseFeaturedItemDTO.getContent()); + dataverseFeaturedItem.setDisplayOrder(newDataverseFeaturedItemDTO.getDisplayOrder()); + dataverseFeaturedItem.setDataverse(dataverse); - private void setImageIfAvailable(DataverseFeaturedItem featuredItem, CommandContext ctxt) throws CommandException { - String imageFileName = newDataverseFeaturedItemDTO.getImageFileName(); - InputStream imageFileInputStream = newDataverseFeaturedItemDTO.getImageFileInputStream(); - if (imageFileName != null && imageFileInputStream != null) { - try { - ctxt.dataverseFeaturedItems().saveDataverseFeaturedItemImageFile(imageFileInputStream, imageFileName, dataverse.getId()); - } catch (DataverseFeaturedItemServiceBean.InvalidImageFileException e) { - throw new InvalidCommandArgumentsException( - e.getMessage(), - this - ); - } catch (IOException e) { - throw new CommandException( - BundleUtil.getStringFromBundle( - "dataverse.create.featuredItem.error.imageFileProcessing", - List.of(e.getMessage()) - ), - this - ); - } - featuredItem.setImageFileName(imageFileName); - } + return ctxt.dataverseFeaturedItems().save(dataverseFeaturedItem); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemCommand.java new file mode 100644 index 00000000000..51e5d43a7aa --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemCommand.java @@ -0,0 +1,41 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.DataverseFeaturedItem; +import edu.harvard.iq.dataverse.api.dto.UpdatedDataverseFeaturedItemDTO; +import edu.harvard.iq.dataverse.engine.command.*; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; + +/** + * Updates a particular featured item {@link DataverseFeaturedItem} of a {@link Dataverse}. + */ +public class UpdateDataverseFeaturedItemCommand extends AbstractWriteDataverseFeaturedItemCommand { + + private final DataverseFeaturedItem dataverseFeaturedItem; + private final UpdatedDataverseFeaturedItemDTO updatedDataverseFeaturedItemDTO; + + public UpdateDataverseFeaturedItemCommand(DataverseRequest request, + DataverseFeaturedItem dataverseFeaturedItem, + UpdatedDataverseFeaturedItemDTO updatedDataverseFeaturedItemDTO) { + super(request, dataverseFeaturedItem.getDataverse()); + this.dataverseFeaturedItem = dataverseFeaturedItem; + this.updatedDataverseFeaturedItemDTO = updatedDataverseFeaturedItemDTO; + } + + @Override + public DataverseFeaturedItem execute(CommandContext ctxt) throws CommandException { + dataverseFeaturedItem.setContent(updatedDataverseFeaturedItemDTO.getContent()); + dataverseFeaturedItem.setDisplayOrder(updatedDataverseFeaturedItemDTO.getDisplayOrder()); + + if (!updatedDataverseFeaturedItemDTO.isKeepFile()) { + setFileImageIfAvailableOrNull( + dataverseFeaturedItem, + updatedDataverseFeaturedItemDTO.getImageFileName(), + updatedDataverseFeaturedItemDTO.getImageFileInputStream(), + ctxt + ); + } + + return ctxt.dataverseFeaturedItems().save(dataverseFeaturedItem); + } +} diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java index 765b1cb25ae..d53414d4207 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java @@ -18,30 +18,20 @@ public static void setUpClass() { @Test public void testDeleteFeaturedItem() { - Response createUserResponse = UtilIT.createRandomUser(); - String apiToken = UtilIT.getApiTokenFromResponse(createUserResponse); - Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); - createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode()); - String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); - - String pathToTestFile = "src/test/resources/images/coffeeshop.png"; - Response createFeatureItemResponse = UtilIT.createDataverseFeaturedItem(dataverseAlias, apiToken, "test", "test", 0, pathToTestFile); - createFeatureItemResponse.then().assertThat().statusCode(OK.getStatusCode()); - - JsonPath createdFeaturedItem = JsonPath.from(createFeatureItemResponse.body().asString()); - Long featuredItemId = createdFeaturedItem.getLong("data.id"); + String apiToken = createUserAndGetApiToken(); + String dataverseAlias = createDataverseAndGetAlias(apiToken); + Long featuredItemId = createFeaturedItemAndGetId(dataverseAlias, apiToken, "src/test/resources/images/coffeeshop.png"); // Should return not found when passing incorrect item id Response deleteFeatureItemResponse = UtilIT.deleteDataverseFeaturedItem(100000L, apiToken); deleteFeatureItemResponse.then().assertThat().statusCode(NOT_FOUND.getStatusCode()); // Should return unauthorized when passing correct id and user does not have permissions - Response createRandomUser = UtilIT.createRandomUser(); - String randomUserApiToken = UtilIT.getApiTokenFromResponse(createRandomUser); + String randomUserApiToken = createUserAndGetApiToken(); deleteFeatureItemResponse = UtilIT.deleteDataverseFeaturedItem(featuredItemId, randomUserApiToken); deleteFeatureItemResponse.then().assertThat().statusCode(UNAUTHORIZED.getStatusCode()); - // Should delete featured item when passing correct id and user have permissions + // Should delete featured item when passing correct id and user has permissions deleteFeatureItemResponse = UtilIT.deleteDataverseFeaturedItem(featuredItemId, apiToken); deleteFeatureItemResponse.then().assertThat().statusCode(OK.getStatusCode()); @@ -50,4 +40,58 @@ public void testDeleteFeaturedItem() { .body("data.size()", equalTo(0)) .assertThat().statusCode(OK.getStatusCode()); } + + @Test + public void testUpdateFeaturedItem() { + String apiToken = createUserAndGetApiToken(); + String dataverseAlias = createDataverseAndGetAlias(apiToken); + Long featuredItemId = createFeaturedItemAndGetId(dataverseAlias, apiToken, "src/test/resources/images/coffeeshop.png"); + + // Should return not found when passing incorrect item id + Response updateFeatureItemResponse = UtilIT.updateDataverseFeaturedItem(100000L, "updatedTitle", 1, false, null, apiToken); + updateFeatureItemResponse.then().assertThat().statusCode(NOT_FOUND.getStatusCode()); + + // Should return unauthorized when passing correct id and user does not have permissions + String randomUserApiToken = createUserAndGetApiToken(); + updateFeatureItemResponse = UtilIT.updateDataverseFeaturedItem(featuredItemId, "updatedTitle", 1, false, null, randomUserApiToken); + updateFeatureItemResponse.then().assertThat().statusCode(UNAUTHORIZED.getStatusCode()); + + // Update featured item: keep image file + updateFeatureItemResponse = UtilIT.updateDataverseFeaturedItem(featuredItemId, "updatedTitle1", 1, true, null, apiToken); + verifyUpdatedFeaturedItem(updateFeatureItemResponse, "updatedTitle1", "coffeeshop.png", 1); + + // Update featured item: remove image file + updateFeatureItemResponse = UtilIT.updateDataverseFeaturedItem(featuredItemId, "updatedTitle1", 2, false, null, apiToken); + verifyUpdatedFeaturedItem(updateFeatureItemResponse, "updatedTitle1", null, 2); + + // Update featured item: set new image file + updateFeatureItemResponse = UtilIT.updateDataverseFeaturedItem(featuredItemId, "updatedTitle1", 2, false, "src/test/resources/images/coffeeshop.png", apiToken); + verifyUpdatedFeaturedItem(updateFeatureItemResponse, "updatedTitle1", "coffeeshop.png", 2); + } + + private String createUserAndGetApiToken() { + Response createUserResponse = UtilIT.createRandomUser(); + return UtilIT.getApiTokenFromResponse(createUserResponse); + } + + private String createDataverseAndGetAlias(String apiToken) { + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode()); + return UtilIT.getAliasFromResponse(createDataverseResponse); + } + + private Long createFeaturedItemAndGetId(String dataverseAlias, String apiToken, String pathToTestFile) { + Response createFeatureItemResponse = UtilIT.createDataverseFeaturedItem(dataverseAlias, apiToken, "test", 0, pathToTestFile); + createFeatureItemResponse.then().assertThat().statusCode(OK.getStatusCode()); + JsonPath createdFeaturedItem = JsonPath.from(createFeatureItemResponse.body().asString()); + return createdFeaturedItem.getLong("data.id"); + } + + private void verifyUpdatedFeaturedItem(Response response, String expectedTitle, String expectedImageFileName, int expectedDisplayOrder) { + response.then().assertThat() + .body("data.content", equalTo(expectedTitle)) + .body("data.imageFileName", equalTo(expectedImageFileName)) + .body("data.displayOrder", equalTo(expectedDisplayOrder)) + .statusCode(OK.getStatusCode()); + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 640e0d42245..8153f7ffe91 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -1586,7 +1586,7 @@ public void testCreateFeaturedItem() { String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); // Should not return any error when not passing a file - Response createFeatureItemResponse = UtilIT.createDataverseFeaturedItem(dataverseAlias, apiToken, "test", "test", 0, null); + Response createFeatureItemResponse = UtilIT.createDataverseFeaturedItem(dataverseAlias, apiToken, "test", 0, null); createFeatureItemResponse.then().assertThat() .statusCode(OK.getStatusCode()) .body("data.content", equalTo("test")) @@ -1595,7 +1595,7 @@ public void testCreateFeaturedItem() { // Should not return any error when passing correct file and data String pathToTestFile = "src/test/resources/images/coffeeshop.png"; - createFeatureItemResponse = UtilIT.createDataverseFeaturedItem(dataverseAlias, apiToken, "test", "test", 1, pathToTestFile); + createFeatureItemResponse = UtilIT.createDataverseFeaturedItem(dataverseAlias, apiToken, "test", 1, pathToTestFile); createFeatureItemResponse.then().assertThat() .body("data.content", equalTo("test")) .body("data.imageFileName", equalTo("coffeeshop.png")) @@ -1604,7 +1604,7 @@ public void testCreateFeaturedItem() { // Should return bad request error when passing incorrect file type pathToTestFile = "src/test/resources/tab/test.tab"; - createFeatureItemResponse = UtilIT.createDataverseFeaturedItem(dataverseAlias, apiToken, "test", "test", 0, pathToTestFile); + createFeatureItemResponse = UtilIT.createDataverseFeaturedItem(dataverseAlias, apiToken, "test", 0, pathToTestFile); createFeatureItemResponse.then().assertThat() .body("message", equalTo(BundleUtil.getStringFromBundle("dataverse.create.featuredItem.error.invalidFileType"))) .statusCode(BAD_REQUEST.getStatusCode()); @@ -1612,11 +1612,11 @@ public void testCreateFeaturedItem() { // Should return unauthorized error when user has no permissions Response createRandomUser = UtilIT.createRandomUser(); String randomUserApiToken = UtilIT.getApiTokenFromResponse(createRandomUser); - createFeatureItemResponse = UtilIT.createDataverseFeaturedItem(dataverseAlias, randomUserApiToken, "test", "test", 0, pathToTestFile); + createFeatureItemResponse = UtilIT.createDataverseFeaturedItem(dataverseAlias, randomUserApiToken, "test", 0, pathToTestFile); createFeatureItemResponse.then().assertThat().statusCode(UNAUTHORIZED.getStatusCode()); // Should return not found error when dataverse does not exist - createFeatureItemResponse = UtilIT.createDataverseFeaturedItem("thisDataverseDoesNotExist", apiToken, "test", "test", 0, pathToTestFile); + createFeatureItemResponse = UtilIT.createDataverseFeaturedItem("thisDataverseDoesNotExist", apiToken, "test", 0, pathToTestFile); createFeatureItemResponse.then().assertThat() .body("message", equalTo("Can't find dataverse with identifier='thisDataverseDoesNotExist'")) .statusCode(NOT_FOUND.getStatusCode()); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 9591f84bdf6..40df922b4fc 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -4360,11 +4360,14 @@ static Response performKeycloakROPCLogin(String username, String password) { .post("http://keycloak.mydomain.com:8090/realms/test/protocol/openid-connect/token"); } - static Response createDataverseFeaturedItem(String dataverseAlias, String apiToken, String title, String content, int displayOrder, String pathToFile) { + static Response createDataverseFeaturedItem(String dataverseAlias, + String apiToken, + String content, + int displayOrder, + String pathToFile) { RequestSpecification requestSpecification = given() .header(API_TOKEN_HTTP_HEADER, apiToken) .contentType(ContentType.MULTIPART) - .multiPart("title", title) .multiPart("content", content) .multiPart("displayOrder", displayOrder); @@ -4383,6 +4386,28 @@ static Response deleteDataverseFeaturedItem(long id, String apiToken) { .delete("/api/dataverseFeaturedItems/" + id); } + static Response updateDataverseFeaturedItem(long featuredItemId, + String content, + int displayOrder, + boolean keepFile, + String pathToFile, + String apiToken) { + RequestSpecification requestSpecification = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .contentType(ContentType.MULTIPART) + .multiPart("content", content) + .multiPart("displayOrder", displayOrder) + .multiPart("keepFile", keepFile); + + if (pathToFile != null) { + requestSpecification.multiPart("file", new File(pathToFile)); + } + + return requestSpecification + .when() + .put("/api/dataverseFeaturedItems/" + featuredItemId); + } + static Response listDataverseFeaturedItems(String dataverseAlias, String apiToken) { return given() .header(API_TOKEN_HTTP_HEADER, apiToken) From 2415ed711ae68428a61e09d3751ed18d586b408e Mon Sep 17 00:00:00 2001 From: Pierre Le Corre <146710476+plecor@users.noreply.github.com> Date: Tue, 7 Jan 2025 08:32:15 +0100 Subject: [PATCH 069/137] Add comment to sql file --- src/main/resources/db/migration/V6.5.0.2.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/resources/db/migration/V6.5.0.2.sql b/src/main/resources/db/migration/V6.5.0.2.sql index c4dbd901181..e2814139e3d 100644 --- a/src/main/resources/db/migration/V6.5.0.2.sql +++ b/src/main/resources/db/migration/V6.5.0.2.sql @@ -1 +1,2 @@ +-- #8739 map publisher tag to distributorName when harvesting update foreignmetadatafieldmapping set datasetfieldname = 'distributorName' where foreignfieldxpath = ':publisher'; From 812ff0f3a2d34356b2af171d7027fccd3c114dbe Mon Sep 17 00:00:00 2001 From: GPortas Date: Tue, 7 Jan 2025 14:21:06 +0000 Subject: [PATCH 070/137] Stash: endpoint for create/update/delete all dv featured items at once WIP --- .../harvard/iq/dataverse/api/Dataverses.java | 68 +++++++++++++++++++ .../UpdateDataverseFeaturedItemsCommand.java | 64 +++++++++++++++++ .../iq/dataverse/api/DataversesIT.java | 23 +++++++ .../edu/harvard/iq/dataverse/api/UtilIT.java | 50 ++++++++++++-- 4 files changed, 199 insertions(+), 6 deletions(-) create mode 100644 src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemsCommand.java diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 015485c9a9e..bf5aff431af 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -66,6 +66,7 @@ import jakarta.ws.rs.WebApplicationException; import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.StreamingOutput; +import org.glassfish.jersey.media.multipart.FormDataBodyPart; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; import org.glassfish.jersey.media.multipart.FormDataParam; @@ -112,6 +113,9 @@ public class Dataverses extends AbstractApiBean { @EJB PermissionServiceBean permissionService; + + @EJB + DataverseFeaturedItemServiceBean dataverseFeaturedItemServiceBean; @POST @AuthRequired @@ -1772,4 +1776,68 @@ public Response listFeaturedItems(@Context ContainerRequestContext crc, @PathPar return e.getResponse(); } } + + // TODO: Refine + @PUT + @AuthRequired + @Consumes(MediaType.MULTIPART_FORM_DATA) + @Path("{dataverseId}/featuredItems") + public Response updateFeaturedItems( + @Context ContainerRequestContext crc, + @PathParam("dataverseId") String dvIdtf, + @FormDataParam("id") List ids, + @FormDataParam("content") List contents, + @FormDataParam("displayOrder") List displayOrders, + @FormDataParam("keepFile") List keepFiles, + @FormDataParam("file") List files) { + + try { + if (contents.size() != displayOrders.size() || displayOrders.size() != files.size()) { + return Response.status(Response.Status.BAD_REQUEST) + .entity("Mismatch between contents, displayOrders, and files.") + .build(); + } + + Dataverse dataverse = findDataverseOrDie(dvIdtf); + List newDataverseFeaturedItemDTOs = new ArrayList<>(); + Map dataverseFeaturedItemsToUpdate = new HashMap<>(); + + for (int i = 0; i < contents.size(); i++) { + Long id = ids.get(i); + String content = contents.get(i); + Integer displayOrder = displayOrders.get(i); + boolean keepFile = keepFiles.get(i); + FormDataBodyPart fileBodyPart = files.get(i); + + InputStream fileInputStream = fileBodyPart.getValueAs(InputStream.class); + FormDataContentDisposition contentDispositionHeader = fileBodyPart.getFormDataContentDisposition(); + + if (id == 0) { + NewDataverseFeaturedItemDTO newDTO = NewDataverseFeaturedItemDTO.fromFormData(content, displayOrder, fileInputStream, contentDispositionHeader); + newDataverseFeaturedItemDTOs.add(newDTO); + } else { + DataverseFeaturedItem existingItem = dataverseFeaturedItemServiceBean.findById(id); + if (existingItem == null) { + return Response.status(Response.Status.NOT_FOUND) + // TODO + .entity("Featured item not found with ID: " + id) + .build(); + } + UpdatedDataverseFeaturedItemDTO updatedDTO = UpdatedDataverseFeaturedItemDTO.fromFormData(content, displayOrder, keepFile, fileInputStream, contentDispositionHeader); + dataverseFeaturedItemsToUpdate.put(existingItem, updatedDTO); + } + } + + execCommand(new UpdateDataverseFeaturedItemsCommand( + createDataverseRequest(getRequestUser(crc)), + dataverse, + newDataverseFeaturedItemDTOs, + dataverseFeaturedItemsToUpdate + )); + + return Response.ok().entity("Featured items updated successfully").build(); + } catch (WrappedResponse wr) { + return wr.getResponse(); + } + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemsCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemsCommand.java new file mode 100644 index 00000000000..a1132188e4b --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemsCommand.java @@ -0,0 +1,64 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.DataverseFeaturedItem; +import edu.harvard.iq.dataverse.api.dto.NewDataverseFeaturedItemDTO; +import edu.harvard.iq.dataverse.api.dto.UpdatedDataverseFeaturedItemDTO; +import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.engine.command.AbstractVoidCommand; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; + +import java.util.List; +import java.util.Map; + +@RequiredPermissions({Permission.EditDataverse}) +public class UpdateDataverseFeaturedItemsCommand extends AbstractVoidCommand { + + private final Dataverse dataverse; + private final List newDataverseFeaturedItemDTOs; + private final Map dataverseFeaturedItemsToUpdate; + + public UpdateDataverseFeaturedItemsCommand(DataverseRequest request, + Dataverse dataverse, + List newDataverseFeaturedItemDTOs, + Map dataverseFeaturedItemsToUpdate) { + super(request, dataverse); + this.dataverse = dataverse; + this.newDataverseFeaturedItemDTOs = newDataverseFeaturedItemDTOs; + this.dataverseFeaturedItemsToUpdate = dataverseFeaturedItemsToUpdate; + } + + @Override + protected void executeImpl(CommandContext ctxt) throws CommandException { + updateOrDeleteExistingFeaturedItems(ctxt); + createNewFeaturedItems(ctxt); + } + + private void updateOrDeleteExistingFeaturedItems(CommandContext ctxt) throws CommandException { + List featuredItemsToDelete = dataverse.getDataverseFeaturedItems(); + + for (Map.Entry entry : dataverseFeaturedItemsToUpdate.entrySet()) { + DataverseFeaturedItem featuredItem = entry.getKey(); + UpdatedDataverseFeaturedItemDTO updatedDTO = entry.getValue(); + + if (featuredItemsToDelete.contains(featuredItem)) { + featuredItemsToDelete.remove(featuredItem); + } + + ctxt.engine().submit(new UpdateDataverseFeaturedItemCommand(getRequest(), featuredItem, updatedDTO)); + } + + for (DataverseFeaturedItem featuredItem : featuredItemsToDelete) { + ctxt.engine().submit(new DeleteDataverseFeaturedItemCommand(getRequest(), featuredItem)); + } + } + + private void createNewFeaturedItems(CommandContext ctxt) throws CommandException { + for (NewDataverseFeaturedItemDTO dto : newDataverseFeaturedItemDTOs) { + ctxt.engine().submit(new CreateDataverseFeaturedItemCommand(getRequest(), dataverse, dto)); + } + } +} diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 8153f7ffe91..51876e13a72 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -1621,4 +1621,27 @@ public void testCreateFeaturedItem() { .body("message", equalTo("Can't find dataverse with identifier='thisDataverseDoesNotExist'")) .statusCode(NOT_FOUND.getStatusCode()); } + + // TODO: Complete + @Test + public void testUpdateFeaturedItems() { + Response createUserResponse = UtilIT.createRandomUser(); + String apiToken = UtilIT.getApiTokenFromResponse(createUserResponse); + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode()); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + List contents = Arrays.asList("Content 1", "Content 2"); + List orders = Arrays.asList(1, 2); + List pathsToFiles = Arrays.asList("src/test/resources/images/coffeeshop.png", "src/test/resources/images/coffeeshop.png"); + + Response updateDataverseFeaturedItemsResponse = UtilIT.updateDataverseFeaturedItems(dataverseAlias, contents, orders, pathsToFiles, apiToken); + updateDataverseFeaturedItemsResponse.prettyPrint(); + updateDataverseFeaturedItemsResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response listFeaturedItemsResponse = UtilIT.listDataverseFeaturedItems(dataverseAlias, apiToken); + listFeaturedItemsResponse.prettyPrint(); + + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 40df922b4fc..2f0417e767d 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -1,12 +1,14 @@ package edu.harvard.iq.dataverse.api; +import com.fasterxml.jackson.databind.ObjectMapper; +import edu.harvard.iq.dataverse.api.dto.NewDataverseFeaturedItemDTO; import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; import io.restassured.http.ContentType; import io.restassured.path.json.JsonPath; import io.restassured.response.Response; import java.io.*; -import java.util.UUID; +import java.util.*; import java.util.logging.Logger; import jakarta.json.Json; import jakarta.json.JsonObjectBuilder; @@ -29,15 +31,13 @@ import org.junit.jupiter.api.Test; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import io.restassured.specification.RequestSpecification; -import java.util.List; import com.mashape.unirest.http.Unirest; import com.mashape.unirest.http.exceptions.UnirestException; import com.mashape.unirest.request.GetRequest; import edu.harvard.iq.dataverse.util.FileUtil; -import java.util.Base64; import org.apache.commons.io.IOUtils; import java.nio.file.Path; -import java.util.ArrayList; + import org.apache.commons.lang3.math.NumberUtils; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; @@ -52,8 +52,7 @@ import edu.harvard.iq.dataverse.settings.FeatureFlags; import edu.harvard.iq.dataverse.util.StringUtil; -import java.util.Collections; - +import static org.apache.http.entity.ContentType.APPLICATION_JSON; import static org.junit.jupiter.api.Assertions.*; public class UtilIT { @@ -4414,4 +4413,43 @@ static Response listDataverseFeaturedItems(String dataverseAlias, String apiToke .contentType("application/json") .get("/api/dataverses/" + dataverseAlias + "/featuredItems"); } + + + // TODO: Refine + static Response updateDataverseFeaturedItems( + String dataverseAlias, + List contents, + List orders, + List pathsToFiles, + String apiToken) { + + RequestSpecification requestSpecification = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .contentType(ContentType.MULTIPART); + + int size = contents.size(); + if (orders.size() != size || pathsToFiles.size() != size) { + throw new IllegalArgumentException("Contents, orders, and pathsToFiles must have the same size."); + } + + for (int i = 0; i < size; i++) { + String content = contents.get(i); + Integer order = orders.get(i); + + requestSpecification.multiPart("content", content); + requestSpecification.multiPart("displayOrder", order); + requestSpecification.multiPart("keepFile", false); + requestSpecification.multiPart("id", 0); + + String pathToFile = pathsToFiles.get(i); + if (pathToFile != null && !pathToFile.isEmpty()) { + requestSpecification.multiPart("file", new File(pathToFile)); + } + } + + return requestSpecification + .when() + .put("/api/dataverses/" + dataverseAlias + "/featuredItems"); + } + } From 791336cd73dea40e4016974c19fdb2b7ffc7505e Mon Sep 17 00:00:00 2001 From: GPortas Date: Tue, 7 Jan 2025 15:40:49 +0000 Subject: [PATCH 071/137] Stash: updateFeaturedItems endpoint WIP. Updated the endpoint to return the latest dataverse featured items after successful update --- .../harvard/iq/dataverse/api/Dataverses.java | 23 ++++++++----------- .../CreateDataverseFeaturedItemCommand.java | 3 +++ .../UpdateDataverseFeaturedItemsCommand.java | 23 +++++++++++-------- src/main/java/propertyFiles/Bundle.properties | 2 ++ .../iq/dataverse/api/DataversesIT.java | 9 ++++---- .../edu/harvard/iq/dataverse/api/UtilIT.java | 15 ++++++------ 6 files changed, 40 insertions(+), 35 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 4550bad41d0..ea7b41b9ca9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -1777,7 +1777,6 @@ public Response listFeaturedItems(@Context ContainerRequestContext crc, @PathPar } } - // TODO: Refine @PUT @AuthRequired @Consumes(MediaType.MULTIPART_FORM_DATA) @@ -1790,12 +1789,14 @@ public Response updateFeaturedItems( @FormDataParam("displayOrder") List displayOrders, @FormDataParam("keepFile") List keepFiles, @FormDataParam("file") List files) { - try { - if (contents.size() != displayOrders.size() || displayOrders.size() != files.size()) { - return Response.status(Response.Status.BAD_REQUEST) - .entity("Mismatch between contents, displayOrders, and files.") - .build(); + if (ids == null || contents == null || displayOrders == null || keepFiles == null || files == null) { + throw new WrappedResponse(error(Response.Status.BAD_REQUEST, BundleUtil.getStringFromBundle("dataverse.update.featuredItems.error.missingInputParams"))); + } + + int size = ids.size(); + if (contents.size() != size || displayOrders.size() != size || keepFiles.size() != size || files.size() != size) { + throw new WrappedResponse(error(Response.Status.BAD_REQUEST, BundleUtil.getStringFromBundle("dataverse.update.featuredItems.error.inputListsSizeMismatch"))); } Dataverse dataverse = findDataverseOrDie(dvIdtf); @@ -1818,24 +1819,20 @@ public Response updateFeaturedItems( } else { DataverseFeaturedItem existingItem = dataverseFeaturedItemServiceBean.findById(id); if (existingItem == null) { - return Response.status(Response.Status.NOT_FOUND) - // TODO - .entity("Featured item not found with ID: " + id) - .build(); + throw new WrappedResponse(error(Response.Status.NOT_FOUND, MessageFormat.format(BundleUtil.getStringFromBundle("dataverseFeaturedItems.errors.notFound"), id))); } UpdatedDataverseFeaturedItemDTO updatedDTO = UpdatedDataverseFeaturedItemDTO.fromFormData(content, displayOrder, keepFile, fileInputStream, contentDispositionHeader); dataverseFeaturedItemsToUpdate.put(existingItem, updatedDTO); } } - execCommand(new UpdateDataverseFeaturedItemsCommand( + List featuredItems = execCommand(new UpdateDataverseFeaturedItemsCommand( createDataverseRequest(getRequestUser(crc)), dataverse, newDataverseFeaturedItemDTOs, dataverseFeaturedItemsToUpdate )); - - return Response.ok().entity("Featured items updated successfully").build(); + return ok(jsonDataverseFeaturedItems(featuredItems)); } catch (WrappedResponse wr) { return wr.getResponse(); } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java index e1ede59817c..e1f21587bb0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java @@ -7,6 +7,9 @@ import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +/** + * Creates a featured item {@link DataverseFeaturedItem} for a {@link Dataverse}. + */ public class CreateDataverseFeaturedItemCommand extends AbstractWriteDataverseFeaturedItemCommand { private final NewDataverseFeaturedItemDTO newDataverseFeaturedItemDTO; diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemsCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemsCommand.java index a1132188e4b..afb09bdf455 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemsCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemsCommand.java @@ -5,26 +5,27 @@ import edu.harvard.iq.dataverse.api.dto.NewDataverseFeaturedItemDTO; import edu.harvard.iq.dataverse.api.dto.UpdatedDataverseFeaturedItemDTO; import edu.harvard.iq.dataverse.authorization.Permission; -import edu.harvard.iq.dataverse.engine.command.AbstractVoidCommand; -import edu.harvard.iq.dataverse.engine.command.CommandContext; -import edu.harvard.iq.dataverse.engine.command.DataverseRequest; -import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; +import edu.harvard.iq.dataverse.engine.command.*; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import java.util.List; import java.util.Map; +/** + * Updates all featured items ({@link DataverseFeaturedItem}) for a specified {@link Dataverse}. + *

+ * This command allows for the creation of multiple new featured items, updates to existing items with new parameters, + * or the deletion of existing items, all in a single command. + *

+ **/ @RequiredPermissions({Permission.EditDataverse}) -public class UpdateDataverseFeaturedItemsCommand extends AbstractVoidCommand { +public class UpdateDataverseFeaturedItemsCommand extends AbstractCommand> { private final Dataverse dataverse; private final List newDataverseFeaturedItemDTOs; private final Map dataverseFeaturedItemsToUpdate; - public UpdateDataverseFeaturedItemsCommand(DataverseRequest request, - Dataverse dataverse, - List newDataverseFeaturedItemDTOs, - Map dataverseFeaturedItemsToUpdate) { + public UpdateDataverseFeaturedItemsCommand(DataverseRequest request, Dataverse dataverse, List newDataverseFeaturedItemDTOs, Map dataverseFeaturedItemsToUpdate) { super(request, dataverse); this.dataverse = dataverse; this.newDataverseFeaturedItemDTOs = newDataverseFeaturedItemDTOs; @@ -32,9 +33,11 @@ public UpdateDataverseFeaturedItemsCommand(DataverseRequest request, } @Override - protected void executeImpl(CommandContext ctxt) throws CommandException { + public List execute(CommandContext ctxt) throws CommandException { updateOrDeleteExistingFeaturedItems(ctxt); createNewFeaturedItems(ctxt); + + return ctxt.engine().submit(new ListDataverseFeaturedItemsCommand(getRequest(), dataverse)); } private void updateOrDeleteExistingFeaturedItems(CommandContext ctxt) throws CommandException { diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index f973cda3770..78adae7062c 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -982,6 +982,8 @@ dataverse.create.error.jsonparsetodataverse=Error parsing the POSTed json into a dataverse.create.featuredItem.error.imageFileProcessing=Error processing featured item file: {0} dataverse.create.featuredItem.error.fileSizeExceedsLimit=File exceeds the maximum size of {0} dataverse.create.featuredItem.error.invalidFileType=Invalid image file type +dataverse.update.featuredItems.error.missingInputParams=All input parameters (id, content, displayOrder, keepFile, file) are required. +dataverse.update.featuredItems.error.inputListsSizeMismatch=All input lists (id, content, displayOrder, keepFile, file) must have the same size. # rolesAndPermissionsFragment.xhtml # advanced.xhtml diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 51876e13a72..6089a44054c 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -1631,17 +1631,16 @@ public void testUpdateFeaturedItems() { createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode()); String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + // Create new items + List ids = Arrays.asList(0L, 0L); List contents = Arrays.asList("Content 1", "Content 2"); List orders = Arrays.asList(1, 2); + List keepFiles = Arrays.asList(false, false); List pathsToFiles = Arrays.asList("src/test/resources/images/coffeeshop.png", "src/test/resources/images/coffeeshop.png"); - Response updateDataverseFeaturedItemsResponse = UtilIT.updateDataverseFeaturedItems(dataverseAlias, contents, orders, pathsToFiles, apiToken); + Response updateDataverseFeaturedItemsResponse = UtilIT.updateDataverseFeaturedItems(dataverseAlias, ids, contents, orders, keepFiles, pathsToFiles, apiToken); updateDataverseFeaturedItemsResponse.prettyPrint(); updateDataverseFeaturedItemsResponse.then().assertThat() .statusCode(OK.getStatusCode()); - - Response listFeaturedItemsResponse = UtilIT.listDataverseFeaturedItems(dataverseAlias, apiToken); - listFeaturedItemsResponse.prettyPrint(); - } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 54dc622bfdd..d0da601eb9a 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -4450,12 +4450,12 @@ static Response listDataverseFeaturedItems(String dataverseAlias, String apiToke .get("/api/dataverses/" + dataverseAlias + "/featuredItems"); } - - // TODO: Refine static Response updateDataverseFeaturedItems( String dataverseAlias, + List ids, List contents, List orders, + List keepFiles, List pathsToFiles, String apiToken) { @@ -4464,18 +4464,20 @@ static Response updateDataverseFeaturedItems( .contentType(ContentType.MULTIPART); int size = contents.size(); - if (orders.size() != size || pathsToFiles.size() != size) { - throw new IllegalArgumentException("Contents, orders, and pathsToFiles must have the same size."); + if (ids.size() != size || orders.size() != size || keepFiles.size() != size || pathsToFiles.size() != size) { + throw new IllegalArgumentException("'ids', 'contents', 'orders', 'keepFiles' and 'pathsToFiles' lists must have the same size."); } for (int i = 0; i < size; i++) { + Long id = ids.get(i); String content = contents.get(i); Integer order = orders.get(i); + boolean keepFile = keepFiles.get(i); requestSpecification.multiPart("content", content); requestSpecification.multiPart("displayOrder", order); - requestSpecification.multiPart("keepFile", false); - requestSpecification.multiPart("id", 0); + requestSpecification.multiPart("keepFile", keepFile); + requestSpecification.multiPart("id", id); String pathToFile = pathsToFiles.get(i); if (pathToFile != null && !pathToFile.isEmpty()) { @@ -4487,5 +4489,4 @@ static Response updateDataverseFeaturedItems( .when() .put("/api/dataverses/" + dataverseAlias + "/featuredItems"); } - } From 48ba8334ff4bb6887c6642e1d8155f678da0dd9e Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 15 Nov 2024 16:11:05 -0500 Subject: [PATCH 072/137] add docs explaining how to run update-fields.sh in configbaker #11023 --- .../source/admin/metadatacustomization.rst | 2 +- .../source/container/configbaker-image.rst | 2 +- doc/sphinx-guides/source/developers/tips.rst | 17 +++++++++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/admin/metadatacustomization.rst b/doc/sphinx-guides/source/admin/metadatacustomization.rst index 4c9dc693a0d..3b1dbf2c8ef 100644 --- a/doc/sphinx-guides/source/admin/metadatacustomization.rst +++ b/doc/sphinx-guides/source/admin/metadatacustomization.rst @@ -544,7 +544,7 @@ a necessary re-index, but for your custom metadata you will need to keep track o Please note also that if you are going to make a pull request updating ``conf/solr/schema.xml`` with fields you have added, you should first load all the custom metadata blocks in ``scripts/api/data/metadatablocks`` (including ones you -don't care about) to create a complete list of fields. (This might change in the future.) +don't care about) to create a complete list of fields. (This might change in the future.) Please see :ref:`update-solr-schema-dev` in the Developer Guide. Reloading a Metadata Block -------------------------- diff --git a/doc/sphinx-guides/source/container/configbaker-image.rst b/doc/sphinx-guides/source/container/configbaker-image.rst index d098bd46436..09e431eb547 100644 --- a/doc/sphinx-guides/source/container/configbaker-image.rst +++ b/doc/sphinx-guides/source/container/configbaker-image.rst @@ -54,7 +54,7 @@ Scripts - Default script when running container without parameters. Lists available scripts and details about them. * - ``update-fields.sh`` - Update a Solr ``schema.xml`` with a given list of metadata fields. See ``update-fields.sh -h`` for usage details - and :ref:`update-solr-schema` for an example use case. + and example use cases at :ref:`update-solr-schema` and :ref:`update-solr-schema-dev`. Solr Template ^^^^^^^^^^^^^ diff --git a/doc/sphinx-guides/source/developers/tips.rst b/doc/sphinx-guides/source/developers/tips.rst index f5ffbac0c07..a0e92015cbd 100755 --- a/doc/sphinx-guides/source/developers/tips.rst +++ b/doc/sphinx-guides/source/developers/tips.rst @@ -187,6 +187,23 @@ Once some Dataverse collections, datasets, and files have been created and index You can simply double-click "start.jar" rather that running ``java -jar start.jar`` from the command line. Figuring out how to stop Solr after double-clicking it is an exercise for the reader. +.. _update-solr-schema-dev: + +Updating the Solr Schema (Developers) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Both developers and sysadmins need to update the Solr schema from time to time. One difference is that developers will be committing changes to ``conf/solr/schema.xml`` in git. To prevent cross-platform differences in the git history, when running the ``update-fields.sh`` script, we ask all developers to run the script from within Docker. (See :doc:`/container/configbaker-image` for more on the image we'll use below.) + +.. code-block:: + + curl http://localhost:8080/api/admin/index/solr/schema | docker run -i --rm -v ./docker-dev-volumes/solr/data:/var/solr gdcc/configbaker:unstable update-fields.sh /var/solr/data/collection1/conf/schema.xml + + cp docker-dev-volumes/solr/data/data/collection1/conf/schema.xml conf/solr/schema.xml + +At this point you can do a ``git diff`` and see if your changes make sense before committing. + +Sysadmins are welcome to run ``update-fields.sh`` however they like. See :ref:`update-solr-schema` in the Admin Guide for details. + Git --- From a9b221653432f990946490a33262b12dedad7ca8 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 7 Jan 2025 16:52:38 -0500 Subject: [PATCH 073/137] explicitly indicate that reordering is supported #11075 --- doc/sphinx-guides/source/admin/metadatacustomization.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/admin/metadatacustomization.rst b/doc/sphinx-guides/source/admin/metadatacustomization.rst index e5326efebef..fed5ab67d26 100644 --- a/doc/sphinx-guides/source/admin/metadatacustomization.rst +++ b/doc/sphinx-guides/source/admin/metadatacustomization.rst @@ -270,7 +270,11 @@ Each of the three main sections own sets of properties: | | โ€œValueโ€ field is used as the identifier. | | +--------------+--------------------------------------------+-----------------------------------------+ | displayOrder | Control the order in which the enumerated | Non-negative integer. | -| | values are displayed for selection. | | +| | values are displayed for selection. When | | +| | adding new values, you don't have to add | | +| | them at the end. You can renumber existing | | +| | values to update the order in which they | | +| | appear. | | +--------------+--------------------------------------------+-----------------------------------------+ FieldType definitions From b1db18e6ddfe480af331788f7131053919948739 Mon Sep 17 00:00:00 2001 From: Pierre Le Corre <146710476+plecor@users.noreply.github.com> Date: Wed, 8 Jan 2025 08:29:06 +0100 Subject: [PATCH 074/137] Update Dataverse version in changelog Co-authored-by: Omer Fahim --- doc/sphinx-guides/source/admin/harvestclients.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/admin/harvestclients.rst b/doc/sphinx-guides/source/admin/harvestclients.rst index 41348cc9774..38a00d6921c 100644 --- a/doc/sphinx-guides/source/admin/harvestclients.rst +++ b/doc/sphinx-guides/source/admin/harvestclients.rst @@ -51,7 +51,7 @@ Note that you'll want to run a minimum of Dataverse Software 4.6, optimally 4.18 Harvesting Client Changelog --------------------------- -- As of Dataverse 6.6, the publisher value of harvested datasets is now attributed to the dataset's distributor instead of its producer. This change affects all newly harvested datasets. For more information, see https://github.com/IQSS/dataverse/pull/9013 +- As of Dataverse 6.5, the publisher value of harvested datasets is now attributed to the dataset's distributor instead of its producer. This change affects all newly harvested datasets. For more information, see https://github.com/IQSS/dataverse/pull/9013 Harvesting Non-OAI-PMH ~~~~~~~~~~~~~~~~~~~~~~ From c9ddbca8f81311c8beaf6a68608e46b158249e4f Mon Sep 17 00:00:00 2001 From: stevenferey Date: Wed, 8 Jan 2025 17:08:19 +0100 Subject: [PATCH 075/137] refactorings and improvements --- .../10304-add-move-dataverse-collection.md | 2 +- .../edu/harvard/iq/dataverse/Dataset.java | 2 + .../iq/dataverse/DatasetServiceBean.java | 8 ++ .../edu/harvard/iq/dataverse/Dataverse.java | 3 +- .../iq/dataverse/DataverseServiceBean.java | 8 ++ ...age.java => DashboardMoveDatasetPage.java} | 37 ++++----- ...e.java => DashboardMoveDataversePage.java} | 31 +++---- .../command/impl/MoveDatasetCommand.java | 8 +- src/main/java/propertyFiles/Bundle.properties | 82 +++++++++---------- src/main/webapp/dashboard-datasetmove.xhtml | 58 ++++++------- src/main/webapp/dashboard-dataversemove.xhtml | 46 +++++------ src/main/webapp/dashboard.xhtml | 10 +-- 12 files changed, 150 insertions(+), 145 deletions(-) rename src/main/java/edu/harvard/iq/dataverse/dashboard/{DashboardDatasetmovePage.java => DashboardMoveDatasetPage.java} (81%) rename src/main/java/edu/harvard/iq/dataverse/dashboard/{DashboardDataversemovePage.java => DashboardMoveDataversePage.java} (82%) diff --git a/doc/release-notes/10304-add-move-dataverse-collection.md b/doc/release-notes/10304-add-move-dataverse-collection.md index 75237743ed7..1ddeb526c61 100644 --- a/doc/release-notes/10304-add-move-dataverse-collection.md +++ b/doc/release-notes/10304-add-move-dataverse-collection.md @@ -1,5 +1,5 @@ ### Move a collection from the dashboard -It is possible for a Dataverse administrator to move a collection from the Dataverse dashboard. +In addition to the api [Move a Dataverse Collection](https://guides.dataverse.org/en/latest/admin/dataverses-datasets.html#move-a-dataverse-collection), it is now possible for a Dataverse administrator to move a collection from the Dataverse dashboard. For more information, see #10304. \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataset.java b/src/main/java/edu/harvard/iq/dataverse/Dataset.java index 78579b1de21..79c64d03d60 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Dataset.java +++ b/src/main/java/edu/harvard/iq/dataverse/Dataset.java @@ -68,6 +68,8 @@ query = "SELECT o FROM Dataset o WHERE o.creator.id=:creatorId"), @NamedQuery(name = "Dataset.findByReleaseUserId", query = "SELECT o FROM Dataset o WHERE o.releaseUser.id=:releaseUserId"), + @NamedQuery(name = "Dataset.countAll", + query = "SELECT COUNT(ds) FROM Dataset ds") }) /* diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java index e519614ba55..9a8c43668cb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java @@ -1092,4 +1092,12 @@ public List getVersionStates(long id) { } } + /** + * Returns the total number of Datasets. + * @return the number of datasets in the database + */ + public long getDatasetCount() { + return em.createNamedQuery("Dataset.countAll", Long.class).getSingleResult(); + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java index 1f11725e581..0ed1fb5c9d7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java +++ b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java @@ -54,7 +54,8 @@ @NamedQuery(name = "Dataverse.findByReleaseUserId", query="select object(o) from Dataverse as o where o.releaseUser.id =:releaseUserId order by o.name"), @NamedQuery(name = "Dataverse.filterByAlias", query="SELECT dv FROM Dataverse dv WHERE LOWER(dv.alias) LIKE :alias order by dv.alias"), @NamedQuery(name = "Dataverse.filterByAliasNameAffiliation", query="SELECT dv FROM Dataverse dv WHERE (LOWER(dv.alias) LIKE :alias) OR (LOWER(dv.name) LIKE :name) OR (LOWER(dv.affiliation) LIKE :affiliation) order by dv.alias"), - @NamedQuery(name = "Dataverse.filterByName", query="SELECT dv FROM Dataverse dv WHERE LOWER(dv.name) LIKE :name order by dv.alias") + @NamedQuery(name = "Dataverse.filterByName", query="SELECT dv FROM Dataverse dv WHERE LOWER(dv.name) LIKE :name order by dv.alias"), + @NamedQuery(name = "Dataverse.countAll", query = "SELECT COUNT(dv) FROM Dataverse dv") }) @Entity @Table(indexes = {@Index(columnList="defaultcontributorrole_id") diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java index 91b15f77111..c407aeb294f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java @@ -1219,4 +1219,12 @@ public void disableStorageQuota(StorageQuota storageQuota) { em.flush(); } } + + /** + * Returns the total number of Dataverses + * @return the number of dataverse in the database + */ + public long getDataverseCount() { + return em.createNamedQuery("Dataverse.countAll", Long.class).getSingleResult(); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/dashboard/DashboardDatasetmovePage.java b/src/main/java/edu/harvard/iq/dataverse/dashboard/DashboardMoveDatasetPage.java similarity index 81% rename from src/main/java/edu/harvard/iq/dataverse/dashboard/DashboardDatasetmovePage.java rename to src/main/java/edu/harvard/iq/dataverse/dashboard/DashboardMoveDatasetPage.java index e3c1a2ad84e..b1333b02a46 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dashboard/DashboardDatasetmovePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/dashboard/DashboardMoveDatasetPage.java @@ -28,13 +28,11 @@ import jakarta.faces.view.ViewScoped; import jakarta.inject.Inject; import jakarta.inject.Named; -import jakarta.persistence.EntityManager; -import jakarta.persistence.PersistenceContext; import jakarta.servlet.http.HttpServletRequest; @ViewScoped -@Named("DashboardDatasetmovePage") -public class DashboardDatasetmovePage implements java.io.Serializable { +@Named("DashboardMoveDatasetPage") +public class DashboardMoveDatasetPage implements java.io.Serializable { @Inject DataverseSession session; @@ -49,11 +47,8 @@ public class DashboardDatasetmovePage implements java.io.Serializable { DataverseServiceBean dataverseService; @Inject SettingsWrapper settingsWrapper; - - @PersistenceContext(unitName = "VDCNet-ejbPU") - private EntityManager em; - private static final Logger logger = Logger.getLogger(DashboardDatasetmovePage.class.getCanonicalName()); + private static final Logger logger = Logger.getLogger(DashboardMoveDatasetPage.class.getCanonicalName()); private AuthenticatedUser authUser = null; @@ -122,18 +117,18 @@ public String init() { FacesContext.getCurrentInstance().addMessage(null, new FacesMessage(FacesMessage.SEVERITY_INFO, - BundleUtil.getStringFromBundle("dashboard.card.datasetmove.manage"), - BundleUtil.getStringFromBundle("dashboard.card.datasetmove.message", Arrays.asList(settingsWrapper.getGuidesBaseUrl(), settingsWrapper.getGuidesVersion())))); + BundleUtil.getStringFromBundle("dashboard.card.move.dataset.manage"), + BundleUtil.getStringFromBundle("dashboard.move.dataset.message", Arrays.asList(settingsWrapper.getGuidesBaseUrl(), settingsWrapper.getGuidesVersion())))); return null; } public void move(){ Dataset ds = selectedSourceDataset; - String dsPersistentId = ds!=null?ds.getGlobalId().asString():null; - String srcAlias = ds!=null?ds.getOwner().getAlias():null; + String dsPersistentId = ds != null ? ds.getGlobalId().asString() : null; + String srcAlias = ds != null ? ds.getOwner().getAlias() : null; Dataverse target = selectedDestinationDataverse; - String dstAlias = target!=null?target.getAlias():null; + String dstAlias = target != null ? target.getAlias() : null; if (ds == null || target == null) { // Move only works if both inputs are correct @@ -148,9 +143,9 @@ public void move(){ // construct arguments for message List arguments = new ArrayList<>(); - arguments.add(ds!=null?ds.getDisplayName():"-"); - arguments.add(dsPersistentId!=null?dsPersistentId:"-"); - arguments.add(target!=null?target.getName():"-"); + arguments.add(ds != null ? ds.getDisplayName() : "-"); + arguments.add(dsPersistentId != null ? dsPersistentId : "-"); + arguments.add(target != null ? target.getName() : "-"); // copied logic from Datasets API move //Command requires Super user - it will be tested by the command @@ -163,7 +158,7 @@ public void move(){ logger.info("Moved " + dsPersistentId + " from " + srcAlias + " to " + dstAlias); - JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dashboard.card.datasetmove.message.success", arguments)); + JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dashboard.move.dataset.message.success", arguments)); } catch (CommandException e) { logger.log(Level.SEVERE,"Unable to move "+ dsPersistentId + " from " + srcAlias + " to " + dstAlias, e); @@ -172,20 +167,20 @@ public void move(){ String guidesBaseUrl = settingsWrapper.getGuidesBaseUrl(); String version = settingsWrapper.getGuidesVersion(); // Suggest using the API to force the move. - arguments.add(BundleUtil.getStringFromBundle("dashboard.card.datasetmove.dataset.command.error.unforced.suggestForce", Arrays.asList(guidesBaseUrl, version))); + arguments.add(BundleUtil.getStringFromBundle("dashboard.move.dataset.command.error.unforced.suggestForce", Arrays.asList(guidesBaseUrl, version))); } else { String emptyStringNoDetails = ""; arguments.add(emptyStringNoDetails); } FacesContext.getCurrentInstance().addMessage(null, new FacesMessage(FacesMessage.SEVERITY_ERROR, - BundleUtil.getStringFromBundle("dashboard.card.datasetmove.message.failure.summary"), - BundleUtil.getStringFromBundle("dashboard.card.datasetmove.message.failure.details", arguments))); + BundleUtil.getStringFromBundle("dashboard.move.dataset.message.failure.summary"), + BundleUtil.getStringFromBundle("dashboard.move.dataset.message.failure.details", arguments))); } } public String getDatasetCount() { - long count = em.createQuery("SELECT count(ds) FROM Dataset ds", Long.class).getSingleResult(); + long count = datasetService.getDatasetCount(); return NumberFormat.getInstance().format(count); } diff --git a/src/main/java/edu/harvard/iq/dataverse/dashboard/DashboardDataversemovePage.java b/src/main/java/edu/harvard/iq/dataverse/dashboard/DashboardMoveDataversePage.java similarity index 82% rename from src/main/java/edu/harvard/iq/dataverse/dashboard/DashboardDataversemovePage.java rename to src/main/java/edu/harvard/iq/dataverse/dashboard/DashboardMoveDataversePage.java index 152d38c25e7..ad67379cf69 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dashboard/DashboardDataversemovePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/dashboard/DashboardMoveDataversePage.java @@ -25,13 +25,11 @@ import jakarta.faces.view.ViewScoped; import jakarta.inject.Inject; import jakarta.inject.Named; -import jakarta.persistence.EntityManager; -import jakarta.persistence.PersistenceContext; import jakarta.servlet.http.HttpServletRequest; @ViewScoped -@Named("DashboardDataversemovePage") -public class DashboardDataversemovePage implements java.io.Serializable { +@Named("DashboardMoveDataversePage") +public class DashboardMoveDataversePage implements java.io.Serializable { @Inject DataverseSession session; @@ -43,11 +41,8 @@ public class DashboardDataversemovePage implements java.io.Serializable { DataverseServiceBean dataverseService; @Inject SettingsWrapper settingsWrapper; - - @PersistenceContext(unitName = "VDCNet-ejbPU") - private EntityManager em; - private static final Logger logger = Logger.getLogger(DashboardDataversemovePage.class.getCanonicalName()); + private static final Logger logger = Logger.getLogger(DashboardMoveDataversePage.class.getCanonicalName()); private AuthenticatedUser authUser = null; @@ -111,17 +106,17 @@ public String init() { FacesContext.getCurrentInstance().addMessage(null, new FacesMessage(FacesMessage.SEVERITY_INFO, - BundleUtil.getStringFromBundle("dashboard.card.dataversemove.message.summary"), - BundleUtil.getStringFromBundle("dashboard.card.dataversemove.message.detail", Arrays.asList(settingsWrapper.getGuidesBaseUrl(), settingsWrapper.getGuidesVersion())))); + BundleUtil.getStringFromBundle("dashboard.card.move.dataverse.message.summary"), + BundleUtil.getStringFromBundle("dashboard.card.move.dataverse.message.detail", Arrays.asList(settingsWrapper.getGuidesBaseUrl(), settingsWrapper.getGuidesVersion())))); return null; } public void move(){ Dataverse dvSource = selectedSourceDataverse; - String srcAlias = dvSource!=null?dvSource.getAlias():null; + String srcAlias = dvSource != null ? dvSource.getAlias() : null; Dataverse target = selectedDestinationDataverse; - String dstAlias = target!=null?target.getAlias():null; + String dstAlias = target !=null ? target.getAlias() : null; if (dvSource == null || target == null) { // Move only works if both inputs are correct @@ -134,8 +129,8 @@ public void move(){ // construct arguments for message List arguments = new ArrayList<>(); - arguments.add(dvSource!=null?dvSource.getName():"-"); - arguments.add(target!=null?target.getName():"-"); + arguments.add(dvSource !=null ? dvSource.getName() : "-"); + arguments.add(target != null ? target.getName() : "-"); // copied logic from Dataverse API move //Command requires Super user - it will be tested by the command @@ -148,20 +143,20 @@ public void move(){ logger.info("Moved " + srcAlias + " to " + dstAlias); - JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dashboard.card.dataversemove.message.success", arguments)); + JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dashboard.card.move.dataverse.message.success", arguments)); } catch (CommandException e) { logger.log(Level.SEVERE,"Unable to move "+ srcAlias + " to " + dstAlias, e); arguments.add(e.getLocalizedMessage()); FacesContext.getCurrentInstance().addMessage(null, new FacesMessage(FacesMessage.SEVERITY_ERROR, - BundleUtil.getStringFromBundle("dashboard.card.dataversemove.message.failure.summary"), - BundleUtil.getStringFromBundle("dashboard.card.dataversemove.message.failure.details", arguments))); + BundleUtil.getStringFromBundle("dashboard.card.move.dataverse.message.failure.summary"), + BundleUtil.getStringFromBundle("dashboard.card.move.dataverse.message.failure.details", arguments))); } } public String getDataverseCount() { - long count = em.createQuery("SELECT count(dv) FROM Dataverse dv", Long.class).getSingleResult(); + long count = dataverseService.getDataverseCount(); return NumberFormat.getInstance().format(count); } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MoveDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MoveDatasetCommand.java index b6a2793b8e0..1c3a62ec6de 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MoveDatasetCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MoveDatasetCommand.java @@ -70,13 +70,13 @@ public void executeImpl(CommandContext ctxt) throws CommandException { // validate the move makes sense if (moved.getOwner().equals(destination)) { - throw new IllegalCommandException(BundleUtil.getStringFromBundle("dashboard.card.datasetmove.dataset.command.error.targetDataverseSameAsOriginalDataverse"), this); + throw new IllegalCommandException(BundleUtil.getStringFromBundle("dashboard.move.dataset.command.error.targetDataverseSameAsOriginalDataverse"), this); } // if dataset is published make sure that its target is published if (moved.isReleased() && !destination.isReleased()){ - throw new IllegalCommandException(BundleUtil.getStringFromBundle("dashboard.card.datasetmove.dataset.command.error.targetDataverseUnpublishedDatasetPublished", Arrays.asList(destination.getDisplayName())), this); + throw new IllegalCommandException(BundleUtil.getStringFromBundle("dashboard.move.dataset.command.error.targetDataverseUnpublishedDatasetPublished", Arrays.asList(destination.getDisplayName())), this); } //if the datasets guestbook is not contained in the new dataverse then remove it @@ -128,10 +128,10 @@ public void executeImpl(CommandContext ctxt) throws CommandException { if (removeGuestbook || removeLinkDs) { StringBuilder errorString = new StringBuilder(); if (removeGuestbook) { - errorString.append(BundleUtil.getStringFromBundle("dashboard.card.datasetmove.dataset.command.error.unforced.datasetGuestbookNotInTargetDataverse")); + errorString.append(BundleUtil.getStringFromBundle("dashboard.move.dataset.command.error.unforced.datasetGuestbookNotInTargetDataverse")); } if (removeLinkDs) { - errorString.append(BundleUtil.getStringFromBundle("dashboard.card.datasetmove.dataset.command.error.unforced.linkedToTargetDataverseOrOneOfItsParents")); + errorString.append(BundleUtil.getStringFromBundle("dashboard.move.dataset.command.error.unforced.linkedToTargetDataverseOrOneOfItsParents")); } throw new UnforcedCommandException(errorString.toString(), this); } diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 009d43ae8c1..c0479c78426 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -542,7 +542,9 @@ dashboard.card.harvestingserver.sets={0, choice, 0#Sets|1#Set|2#Sets} dashboard.card.harvestingserver.btn.manage=Manage Server dashboard.card.metadataexport.header=Metadata Export dashboard.card.metadataexport.message=Dataset metadata export is only available through the {0} API. Learn more in the {0} {1}API Guide{2}. -dashboard.card.datamove=Data +dashboard.card.move.data=Data +dashboard.card.move.dataset.manage=Move Dataset +dashboard.card.move.dataverse.manage=Move Collection #harvestclients.xhtml harvestclients.title=Manage Harvesting Clients @@ -749,50 +751,44 @@ dashboard.list_users.api.auth.invalid_apikey=The API key is invalid. dashboard.list_users.api.auth.not_superuser=Forbidden. You must be a superuser. #dashboard-datasetmove.xhtml -dashboard.card.datasetmove.header=Dashboard - Move Data -dashboard.card.datasetmove.manage=Move Dataset -dashboard.card.datasetmove.message=Manage and curate your installation by moving datasets from one host dataverse to another. See also Managing Datasets and Dataverses in the Admin Guide. -dashboard.card.datasetmove.selectdataset.header=Dataset to Move -dashboard.card.datasetmove.newdataverse.header=New Host Dataverse -dashboard.card.datasetmove.dataset.label=Dataset -dashboard.card.datasetmove.dataverse.label=Dataverse -dashboard.card.datasetmove.confirm.dialog=Are you sure want to move this dataset? -dashboard.card.datasetmove.confirm.yes=Yes, Move Dataset -dashboard.card.datsetamove.message.success=The dataset "{0}" ({1}) has been successfully moved to {2}. -dashboard.card.datasetmove.message.failure.summary=Failed to moved dataset -dashboard.card.datasetmove.message.failure.details=The dataset "{0}" ({1}) could not be moved to {2}. {3}{4} -dashboard.card.datasetmove.dataverse.placeholder=Enter Dataverse Identifier... -dashboard.card.datasetmove.dataverse.menu.header=Dataverse Name (Affiliate), Identifier -dashboard.card.datasetmove.dataverse.menu.invalidMsg=No matches found -dashboard.card.datasetmove.dataset.placeholder=Enter Dataset Persistent ID, doi:... -dashboard.card.datasetmove.dataset.menu.header=Dataset Persistent ID, Title, Host Dataverse Identifier -dashboard.card.datasetmove.dataset.menu.invalidMsg=No matches found -dashboard.card.datasetmove.dataset.command.error.targetDataverseUnpublishedDatasetPublished=A published dataset may not be moved to an unpublished dataverse. You can retry the move after publishing {0}. -dashboard.card.datasetmove.dataset.command.error.targetDataverseSameAsOriginalDataverse=This dataset is already in this dataverse. -dashboard.card.datasetmove.dataset.command.error.unforced.datasetGuestbookNotInTargetDataverse=The guestbook would be removed from this dataset if you moved it because the guestbook is not in the new host dataverse. -dashboard.card.datasetmove.dataset.command.error.unforced.linkedToTargetDataverseOrOneOfItsParents=This dataset is linked to the new host dataverse or one of its parents. This move would remove the link to this dataset. -dashboard.card.datasetmove.dataset.command.error.unforced.suggestForce=Forcing this move is currently only available via API. Please see "Move a Dataset" under Managing Datasets and Dataverses in the Admin Guide for details. +dashboard.move.dataset.header=Dashboard - Move Data +dashboard.move.dataset.message=Manage and curate your installation by moving datasets from one host dataverse to another. See also Managing Datasets and Dataverses in the Admin Guide. +dashboard.move.dataset.selectdataset.header=Dataset to Move +dashboard.move.dataset.newdataverse.header=New Host Dataverse +dashboard.move.dataset.dataset.label=Dataset +dashboard.move.dataset.dataverse.label=Dataverse +dashboard.move.dataset.confirm.dialog=Are you sure want to move this dataset? +dashboard.move.dataset.confirm.yes=Yes, Move Dataset +dashboard.move.dataset.message.success=The dataset "{0}" ({1}) has been successfully moved to {2}. +dashboard.move.dataset.message.failure.summary=Failed to moved dataset +dashboard.move.dataset.message.failure.details=The dataset "{0}" ({1}) could not be moved to {2}. {3}{4} +dashboard.move.dataset.dataverse.placeholder=Enter Dataverse Identifier... +dashboard.move.dataset.dataverse.menu.header=Dataverse Name (Affiliate), Identifier +dashboard.move.dataset.dataverse.menu.invalidMsg=No matches found +dashboard.move.dataset.placeholder=Enter Dataset Persistent ID, doi:... +dashboard.move.dataset.menu.header=Dataset Persistent ID, Title, Host Dataverse Identifier +dashboard.move.dataset.menu.invalidMsg=No matches found +dashboard.move.dataset.command.error.targetDataverseUnpublishedDatasetPublished=A published dataset may not be moved to an unpublished dataverse. You can retry the move after publishing {0}. +dashboard.move.dataset.command.error.targetDataverseSameAsOriginalDataverse=This dataset is already in this dataverse. +dashboard.move.dataset.command.error.unforced.datasetGuestbookNotInTargetDataverse=The guestbook would be removed from this dataset if you moved it because the guestbook is not in the new host dataverse. +dashboard.move.dataset.command.error.unforced.linkedToTargetDataverseOrOneOfItsParents=This dataset is linked to the new host dataverse or one of its parents. This move would remove the link to this dataset. +dashboard.move.dataset.command.error.unforced.suggestForce=Forcing this move is currently only available via API. Please see "Move a Dataset" under Managing Datasets and Dataverses in the Admin Guide for details. #dashboard-dataversemove.xhtml -dashboard.card.dataversemove.header=Dashboard - Move Data -dashboard.card.dataversemove.manage=Move Collection -dashboard.card.dataversemove.message.summary=Move Dataverse Collection -dashboard.card.dataversemove.message.detail=Manage and curate your installation by moving a dataverse collection from one host dataverse collection to another. See also Managing Datasets and Dataverses in the Admin Guide. -dashboard.card.dataversemove.selectdataverse.header=Dataverse to Move -dashboard.card.dataversemove.newdataverse.header=New Host Dataverse -dashboard.card.dataversemove.dataverse.label=Dataverse -dashboard.card.dataversemove.confirm.dialog=Are you sure want to move this dataverse? -dashboard.card.dataversemove.confirm.yes=Yes, Move Dataverse -dashboard.card.dataversemove.message.success=The dataverse "{0}" has been successfully moved to {1}. -dashboard.card.dataversemove.message.failure.summary=Failed to moved dataverse -dashboard.card.dataversemove.message.failure.details=The dataverse "{0}" could not be moved to {1}. {2} -dashboard.card.dataversemove.dataverse.placeholder=Enter Dataverse Identifier... -dashboard.card.dataversemove.dataverse.menu.header=Dataverse Name (Affiliate), Identifier -dashboard.card.dataversemove.dataverse.menu.invalidMsg=No matches found -dashboard.card.dataversemove.dataset.command.error.targetDataverseUnpublishedDatasetPublished=A published dataset may not be moved to an unpublished dataverse. You can retry the move after publishing {0}. -dashboard.card.dataversemove.dataset.command.error.targetDataverseSameAsOriginalDataverse=This dataset is already in this dataverse. -dashboard.card.dataversemove.dataset.command.error.unforced.datasetGuestbookNotInTargetDataverse=The guestbook would be removed from this dataset if you moved it because the guestbook is not in the new host dataverse. -dashboard.card.dataversemove.dataset.command.error.unforced.linkedToTargetDataverseOrOneOfItsParents=This dataset is linked to the new host dataverse or one of its parents. This move would remove the link to this dataset. +dashboard.card.move.dataverse.header=Dashboard - Move Data +dashboard.card.move.dataverse.message.summary=Move Dataverse Collection +dashboard.card.move.dataverse.message.detail=Manage and curate your installation by moving a dataverse collection from one host dataverse collection to another. See also Managing Datasets and Dataverses in the Admin Guide. +dashboard.card.move.dataverse.selectdataverse.header=Dataverse to Move +dashboard.card.move.dataverse.newdataverse.header=New Host Dataverse +dashboard.card.move.dataverse.label=Dataverse +dashboard.card.move.dataverse.confirm.dialog=Are you sure want to move this dataverse? +dashboard.card.move.dataverse.confirm.yes=Yes, Move Dataverse +dashboard.card.move.dataverse.message.success=The dataverse "{0}" has been successfully moved to {1}. +dashboard.card.move.dataverse.message.failure.summary=Failed to moved dataverse +dashboard.card.move.dataverse.message.failure.details=The dataverse "{0}" could not be moved to {1}. {2} +dashboard.card.move.dataverse.placeholder=Enter Dataverse Identifier... +dashboard.card.move.dataverse.menu.header=Dataverse Name (Affiliate), Identifier +dashboard.card.move.dataverse.menu.invalidMsg=No matches found #MailServiceBean.java notification.email.create.dataverse.subject={0}: Your dataverse has been created diff --git a/src/main/webapp/dashboard-datasetmove.xhtml b/src/main/webapp/dashboard-datasetmove.xhtml index 96d9d401412..ded02badef5 100644 --- a/src/main/webapp/dashboard-datasetmove.xhtml +++ b/src/main/webapp/dashboard-datasetmove.xhtml @@ -13,40 +13,40 @@ - + - + - + - +
-
#{bundle['dashboard.card.datasetmove.selectdataset.header']}
+
#{bundle['dashboard.move.dataset.selectdataset.header']}
@@ -70,26 +70,26 @@
-
#{bundle['dashboard.card.datasetmove.newdataverse.header']}
+
#{bundle['dashboard.move.dataset.newdataverse.header']}
@@ -113,12 +113,12 @@
+ oncomplete="if (args && !args.validationFailed) PF('datasetmoveConfirmation').show();"> - +
@@ -68,26 +68,26 @@
-
#{bundle['dashboard.card.dataversemove.newdataverse.header']}
+
#{bundle['dashboard.card.move.dataverse.newdataverse.header']}
@@ -120,12 +120,12 @@
- -

#{bundle['dashboard.card.dataversemove.confirm.dialog']}

+ +

#{bundle['dashboard.card.move.dataverse.confirm.dialog']}

-
diff --git a/src/main/webapp/dashboard.xhtml b/src/main/webapp/dashboard.xhtml index f8bdafe875b..5f083a7455f 100644 --- a/src/main/webapp/dashboard.xhtml +++ b/src/main/webapp/dashboard.xhtml @@ -140,12 +140,12 @@

- #{bundle['dashboard.card.move.dataset.manage']} + #{bundle['dashboard.card.move.dataset.manage']}

- #{bundle['dashboard.card.move.dataverse.manage']} + #{bundle['dashboard.card.move.dataverse.manage']}

From d1c7a040e15daba09385ff16d9540de805a5fcf6 Mon Sep 17 00:00:00 2001 From: GPortas Date: Thu, 9 Jan 2025 14:59:49 +0000 Subject: [PATCH 079/137] Changed: relocated flush call to avoid unnecessary call in DataverseFeaturedItemServiceBean --- .../harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java index 90fb3e04d1f..b5ee7f60166 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java @@ -37,10 +37,10 @@ public DataverseFeaturedItem findById(Long id) { public DataverseFeaturedItem save(DataverseFeaturedItem dataverseFeaturedItem) { if (dataverseFeaturedItem.getId() == null) { em.persist(dataverseFeaturedItem); + em.flush(); } else { dataverseFeaturedItem = em.merge(dataverseFeaturedItem); } - em.flush(); return dataverseFeaturedItem; } From b8853b7a62b0ecdcc7f67c59a2305d66472f9218 Mon Sep 17 00:00:00 2001 From: GPortas Date: Fri, 10 Jan 2025 09:50:25 +0000 Subject: [PATCH 080/137] Fixed: delete featured item inconsistencies --- .../edu/harvard/iq/dataverse/DataverseFeaturedItem.java | 4 ++++ .../iq/dataverse/DataverseFeaturedItemServiceBean.java | 6 ++++++ .../command/impl/DeleteDataverseFeaturedItemCommand.java | 4 +--- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java index 52da2bb9be5..fb2077c5f27 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java @@ -6,6 +6,10 @@ import jakarta.validation.constraints.Min; import jakarta.validation.constraints.Size; +@NamedQueries({ + @NamedQuery(name = "DataverseFeaturedItem.deleteById", + query = "DELETE FROM DataverseFeaturedItem item WHERE item.id=:id") +}) @Entity public class DataverseFeaturedItem { diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java index b5ee7f60166..73f7b5c5057 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java @@ -44,6 +44,12 @@ public DataverseFeaturedItem save(DataverseFeaturedItem dataverseFeaturedItem) { return dataverseFeaturedItem; } + public void delete(Long id) { + em.createNamedQuery("DataverseFeaturedItem.deleteById", DataverseFeaturedItem.class) + .setParameter("id", id) + .executeUpdate(); + } + public InputStream getImageFileAsInputStream(DataverseFeaturedItem dataverseFeaturedItem) throws IOException { Path imagePath = Path.of(JvmSettings.DOCROOT_DIRECTORY.lookup(), JvmSettings.FEATURED_ITEMS_IMAGE_UPLOADS_DIRECTORY.lookup(), diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataverseFeaturedItemCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataverseFeaturedItemCommand.java index 7ed0e60cccb..3a7504b8c77 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataverseFeaturedItemCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataverseFeaturedItemCommand.java @@ -21,8 +21,6 @@ public DeleteDataverseFeaturedItemCommand(DataverseRequest request, DataverseFea @Override protected void executeImpl(CommandContext ctxt) throws CommandException { - ctxt.em().merge(doomed.getDataverse()); - DataverseFeaturedItem doomedAndMerged = ctxt.em().merge(doomed); - ctxt.em().remove(doomedAndMerged); + ctxt.dataverseFeaturedItems().delete(doomed.getId()); } } From 3df4723d5d308ed3593e3d5cdae1ba1092d5c5b2 Mon Sep 17 00:00:00 2001 From: GPortas Date: Fri, 10 Jan 2025 09:51:22 +0000 Subject: [PATCH 081/137] Added: updateFeaturedItems working endpoint --- .../harvard/iq/dataverse/api/Dataverses.java | 59 +++++---- .../UpdateDataverseFeaturedItemsCommand.java | 33 +++-- src/main/java/propertyFiles/Bundle.properties | 4 +- .../iq/dataverse/api/DataversesIT.java | 114 ++++++++++++++++-- .../edu/harvard/iq/dataverse/api/UtilIT.java | 33 ++--- 5 files changed, 178 insertions(+), 65 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index ea7b41b9ca9..d4bbb50b971 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -1788,50 +1788,61 @@ public Response updateFeaturedItems( @FormDataParam("content") List contents, @FormDataParam("displayOrder") List displayOrders, @FormDataParam("keepFile") List keepFiles, + @FormDataParam("fileName") List fileNames, @FormDataParam("file") List files) { try { - if (ids == null || contents == null || displayOrders == null || keepFiles == null || files == null) { - throw new WrappedResponse(error(Response.Status.BAD_REQUEST, BundleUtil.getStringFromBundle("dataverse.update.featuredItems.error.missingInputParams"))); + if (ids == null || contents == null || displayOrders == null || keepFiles == null || fileNames == null) { + throw new WrappedResponse(error(Response.Status.BAD_REQUEST, + BundleUtil.getStringFromBundle("dataverse.update.featuredItems.error.missingInputParams"))); } int size = ids.size(); - if (contents.size() != size || displayOrders.size() != size || keepFiles.size() != size || files.size() != size) { - throw new WrappedResponse(error(Response.Status.BAD_REQUEST, BundleUtil.getStringFromBundle("dataverse.update.featuredItems.error.inputListsSizeMismatch"))); + if (contents.size() != size || displayOrders.size() != size || keepFiles.size() != size || fileNames.size() != size) { + throw new WrappedResponse(error(Response.Status.BAD_REQUEST, + BundleUtil.getStringFromBundle("dataverse.update.featuredItems.error.inputListsSizeMismatch"))); } Dataverse dataverse = findDataverseOrDie(dvIdtf); - List newDataverseFeaturedItemDTOs = new ArrayList<>(); - Map dataverseFeaturedItemsToUpdate = new HashMap<>(); + List newItems = new ArrayList<>(); + Map itemsToUpdate = new HashMap<>(); for (int i = 0; i < contents.size(); i++) { - Long id = ids.get(i); - String content = contents.get(i); - Integer displayOrder = displayOrders.get(i); - boolean keepFile = keepFiles.get(i); - FormDataBodyPart fileBodyPart = files.get(i); - - InputStream fileInputStream = fileBodyPart.getValueAs(InputStream.class); - FormDataContentDisposition contentDispositionHeader = fileBodyPart.getFormDataContentDisposition(); - - if (id == 0) { - NewDataverseFeaturedItemDTO newDTO = NewDataverseFeaturedItemDTO.fromFormData(content, displayOrder, fileInputStream, contentDispositionHeader); - newDataverseFeaturedItemDTOs.add(newDTO); + String fileName = fileNames.get(i); + InputStream fileInputStream = null; + FormDataContentDisposition contentDisposition = null; + + if (files != null) { + Optional matchingFile = files.stream() + .filter(file -> file.getFormDataContentDisposition().getFileName().equals(fileName)) + .findFirst(); + + if (matchingFile.isPresent()) { + fileInputStream = matchingFile.get().getValueAs(InputStream.class); + contentDisposition = matchingFile.get().getFormDataContentDisposition(); + } + } + + if (ids.get(i) == 0) { + newItems.add(NewDataverseFeaturedItemDTO.fromFormData( + contents.get(i), displayOrders.get(i), fileInputStream, contentDisposition)); } else { - DataverseFeaturedItem existingItem = dataverseFeaturedItemServiceBean.findById(id); + DataverseFeaturedItem existingItem = dataverseFeaturedItemServiceBean.findById(ids.get(i)); if (existingItem == null) { - throw new WrappedResponse(error(Response.Status.NOT_FOUND, MessageFormat.format(BundleUtil.getStringFromBundle("dataverseFeaturedItems.errors.notFound"), id))); + throw new WrappedResponse(error(Response.Status.NOT_FOUND, + MessageFormat.format(BundleUtil.getStringFromBundle("dataverseFeaturedItems.errors.notFound"), ids.get(i)))); } - UpdatedDataverseFeaturedItemDTO updatedDTO = UpdatedDataverseFeaturedItemDTO.fromFormData(content, displayOrder, keepFile, fileInputStream, contentDispositionHeader); - dataverseFeaturedItemsToUpdate.put(existingItem, updatedDTO); + itemsToUpdate.put(existingItem, UpdatedDataverseFeaturedItemDTO.fromFormData( + contents.get(i), displayOrders.get(i), keepFiles.get(i), fileInputStream, contentDisposition)); } } List featuredItems = execCommand(new UpdateDataverseFeaturedItemsCommand( createDataverseRequest(getRequestUser(crc)), dataverse, - newDataverseFeaturedItemDTOs, - dataverseFeaturedItemsToUpdate + newItems, + itemsToUpdate )); + return ok(jsonDataverseFeaturedItems(featuredItems)); } catch (WrappedResponse wr) { return wr.getResponse(); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemsCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemsCommand.java index afb09bdf455..8fb5f4f09c8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemsCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemsCommand.java @@ -8,6 +8,8 @@ import edu.harvard.iq.dataverse.engine.command.*; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import java.util.Map; @@ -34,34 +36,43 @@ public UpdateDataverseFeaturedItemsCommand(DataverseRequest request, Dataverse d @Override public List execute(CommandContext ctxt) throws CommandException { - updateOrDeleteExistingFeaturedItems(ctxt); - createNewFeaturedItems(ctxt); - - return ctxt.engine().submit(new ListDataverseFeaturedItemsCommand(getRequest(), dataverse)); + List dataverseFeaturedItems = updateOrDeleteExistingFeaturedItems(ctxt); + dataverseFeaturedItems.addAll(createNewFeaturedItems(ctxt)); + dataverseFeaturedItems.sort(Comparator.comparingLong(DataverseFeaturedItem::getId)); + return dataverseFeaturedItems; } - private void updateOrDeleteExistingFeaturedItems(CommandContext ctxt) throws CommandException { + private List updateOrDeleteExistingFeaturedItems(CommandContext ctxt) throws CommandException { + List updatedFeaturedItems = new ArrayList<>(); List featuredItemsToDelete = dataverse.getDataverseFeaturedItems(); for (Map.Entry entry : dataverseFeaturedItemsToUpdate.entrySet()) { DataverseFeaturedItem featuredItem = entry.getKey(); UpdatedDataverseFeaturedItemDTO updatedDTO = entry.getValue(); - if (featuredItemsToDelete.contains(featuredItem)) { - featuredItemsToDelete.remove(featuredItem); - } + featuredItemsToDelete.stream() + .filter(item -> item.getId().equals(featuredItem.getId())) + .findFirst().ifPresent(featuredItemsToDelete::remove); - ctxt.engine().submit(new UpdateDataverseFeaturedItemCommand(getRequest(), featuredItem, updatedDTO)); + DataverseFeaturedItem updatedFeatureItem = ctxt.engine().submit(new UpdateDataverseFeaturedItemCommand(getRequest(), featuredItem, updatedDTO)); + updatedFeaturedItems.add(updatedFeatureItem); } for (DataverseFeaturedItem featuredItem : featuredItemsToDelete) { ctxt.engine().submit(new DeleteDataverseFeaturedItemCommand(getRequest(), featuredItem)); } + + return updatedFeaturedItems; } - private void createNewFeaturedItems(CommandContext ctxt) throws CommandException { + private List createNewFeaturedItems(CommandContext ctxt) throws CommandException { + List createdFeaturedItems = new ArrayList<>(); + for (NewDataverseFeaturedItemDTO dto : newDataverseFeaturedItemDTOs) { - ctxt.engine().submit(new CreateDataverseFeaturedItemCommand(getRequest(), dataverse, dto)); + DataverseFeaturedItem createdFeatureItem = ctxt.engine().submit(new CreateDataverseFeaturedItemCommand(getRequest(), dataverse, dto)); + createdFeaturedItems.add(createdFeatureItem); } + + return createdFeaturedItems; } } diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 78adae7062c..ab674a9dc15 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -982,8 +982,8 @@ dataverse.create.error.jsonparsetodataverse=Error parsing the POSTed json into a dataverse.create.featuredItem.error.imageFileProcessing=Error processing featured item file: {0} dataverse.create.featuredItem.error.fileSizeExceedsLimit=File exceeds the maximum size of {0} dataverse.create.featuredItem.error.invalidFileType=Invalid image file type -dataverse.update.featuredItems.error.missingInputParams=All input parameters (id, content, displayOrder, keepFile, file) are required. -dataverse.update.featuredItems.error.inputListsSizeMismatch=All input lists (id, content, displayOrder, keepFile, file) must have the same size. +dataverse.update.featuredItems.error.missingInputParams=All input parameters (id, content, displayOrder, keepFile, fileNames) are required. +dataverse.update.featuredItems.error.inputListsSizeMismatch=All input lists (id, content, displayOrder, keepFile, fileNames) must have the same size. # rolesAndPermissionsFragment.xhtml # advanced.xhtml diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 6089a44054c..ea29fb487fc 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -1622,7 +1622,6 @@ public void testCreateFeaturedItem() { .statusCode(NOT_FOUND.getStatusCode()); } - // TODO: Complete @Test public void testUpdateFeaturedItems() { Response createUserResponse = UtilIT.createRandomUser(); @@ -1630,17 +1629,118 @@ public void testUpdateFeaturedItems() { Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode()); String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + String baseUri = UtilIT.getRestAssuredBaseUri(); // Create new items - List ids = Arrays.asList(0L, 0L); - List contents = Arrays.asList("Content 1", "Content 2"); - List orders = Arrays.asList(1, 2); - List keepFiles = Arrays.asList(false, false); - List pathsToFiles = Arrays.asList("src/test/resources/images/coffeeshop.png", "src/test/resources/images/coffeeshop.png"); + + List ids = Arrays.asList(0L, 0L, 0L); + List contents = Arrays.asList("Content 1", "Content 2", "Content 3"); + List orders = Arrays.asList(0, 1, 2); + List keepFiles = Arrays.asList(false, false, false); + List pathsToFiles = Arrays.asList("src/test/resources/images/coffeeshop.png", null, null); Response updateDataverseFeaturedItemsResponse = UtilIT.updateDataverseFeaturedItems(dataverseAlias, ids, contents, orders, keepFiles, pathsToFiles, apiToken); - updateDataverseFeaturedItemsResponse.prettyPrint(); updateDataverseFeaturedItemsResponse.then().assertThat() + .body("data.size()", equalTo(3)) + .body("data[0].content", equalTo("Content 1")) + .body("data[0].imageFileName", equalTo("coffeeshop.png")) + .body("data[0].imageFileUrl", containsString(baseUri + "/api/access/dataverseFeatureItemImage/")) + .body("data[0].displayOrder", equalTo(0)) + .body("data[1].content", equalTo("Content 2")) + .body("data[1].imageFileName", equalTo(null)) + .body("data[1].imageFileUrl", equalTo(null)) + .body("data[1].displayOrder", equalTo(1)) + .body("data[2].content", equalTo("Content 3")) + .body("data[2].imageFileName", equalTo(null)) + .body("data[2].imageFileUrl", equalTo(null)) + .body("data[2].displayOrder", equalTo(2)) + .statusCode(OK.getStatusCode()); + + Long firstItemId = JsonPath.from(updateDataverseFeaturedItemsResponse.body().asString()).getLong("data[0].id"); + Long secondItemId = JsonPath.from(updateDataverseFeaturedItemsResponse.body().asString()).getLong("data[1].id"); + Long thirdItemId = JsonPath.from(updateDataverseFeaturedItemsResponse.body().asString()).getLong("data[2].id"); + + // Update first item (content, order, and keeping image), delete the rest and create new items + + ids = Arrays.asList(firstItemId, 0L, 0L); + contents = Arrays.asList("Content 1 updated", "Content 2", "Content 3"); + orders = Arrays.asList(1, 0, 2); + keepFiles = Arrays.asList(true, false, false); + pathsToFiles = Arrays.asList(null, null, null); + + updateDataverseFeaturedItemsResponse = UtilIT.updateDataverseFeaturedItems(dataverseAlias, ids, contents, orders, keepFiles, pathsToFiles, apiToken); + updateDataverseFeaturedItemsResponse.then().assertThat() + .body("data.size()", equalTo(3)) + .body("data[0].content", equalTo("Content 1 updated")) + .body("data[0].imageFileName", equalTo("coffeeshop.png")) + .body("data[0].imageFileUrl", containsString(baseUri + "/api/access/dataverseFeatureItemImage/")) + .body("data[0].displayOrder", equalTo(1)) + .body("data[1].content", equalTo("Content 2")) + .body("data[1].imageFileName", equalTo(null)) + .body("data[1].imageFileUrl", equalTo(null)) + .body("data[1].displayOrder", equalTo(0)) + .body("data[2].content", equalTo("Content 3")) + .body("data[2].imageFileName", equalTo(null)) + .body("data[2].imageFileUrl", equalTo(null)) + .body("data[2].displayOrder", equalTo(2)) + .statusCode(OK.getStatusCode()); + + Long firstItemIdAfterUpdate = JsonPath.from(updateDataverseFeaturedItemsResponse.body().asString()).getLong("data[0].id"); + Long secondItemIdAfterUpdate = JsonPath.from(updateDataverseFeaturedItemsResponse.body().asString()).getLong("data[1].id"); + Long thirdItemIdAfterUpdate = JsonPath.from(updateDataverseFeaturedItemsResponse.body().asString()).getLong("data[2].id"); + + assertEquals(firstItemId, firstItemIdAfterUpdate); + assertNotEquals(secondItemId, secondItemIdAfterUpdate); + assertNotEquals(thirdItemId, thirdItemIdAfterUpdate); + + // Update first item (removing image), update second item (adding image), delete the third item and create a new item + + ids = Arrays.asList(firstItemId, secondItemIdAfterUpdate, 0L); + contents = Arrays.asList("Content 1 updated", "Content 2", "Content 3"); + orders = Arrays.asList(1, 0, 2); + keepFiles = Arrays.asList(false, false, false); + pathsToFiles = Arrays.asList(null, "src/test/resources/images/coffeeshop.png", null); + + updateDataverseFeaturedItemsResponse = UtilIT.updateDataverseFeaturedItems(dataverseAlias, ids, contents, orders, keepFiles, pathsToFiles, apiToken); + updateDataverseFeaturedItemsResponse.then().assertThat() + .body("data.size()", equalTo(3)) + .body("data[0].content", equalTo("Content 1 updated")) + .body("data[0].imageFileName", equalTo(null)) + .body("data[0].imageFileUrl", equalTo(null)) + .body("data[0].displayOrder", equalTo(1)) + .body("data[1].content", equalTo("Content 2")) + .body("data[1].imageFileName", equalTo("coffeeshop.png")) + .body("data[1].imageFileUrl", containsString(baseUri + "/api/access/dataverseFeatureItemImage/")) + .body("data[1].displayOrder", equalTo(0)) + .body("data[2].content", equalTo("Content 3")) + .body("data[2].imageFileName", equalTo(null)) + .body("data[2].imageFileUrl", equalTo(null)) + .body("data[2].displayOrder", equalTo(2)) + .statusCode(OK.getStatusCode()); + + Long firstItemIdAftersSecondUpdate = JsonPath.from(updateDataverseFeaturedItemsResponse.body().asString()).getLong("data[0].id"); + Long secondItemIdAfterSecondUpdate = JsonPath.from(updateDataverseFeaturedItemsResponse.body().asString()).getLong("data[1].id"); + Long thirdItemIdAfterSecondUpdate = JsonPath.from(updateDataverseFeaturedItemsResponse.body().asString()).getLong("data[2].id"); + + assertEquals(firstItemId, firstItemIdAftersSecondUpdate); + assertEquals(secondItemIdAfterUpdate, secondItemIdAfterSecondUpdate); + assertNotEquals(thirdItemIdAfterUpdate, thirdItemIdAfterSecondUpdate); + + // Only keep first featured item + + ids = List.of(firstItemId); + contents = List.of("Content 1 updated"); + orders = List.of(0); + keepFiles = List.of(false); + pathsToFiles = null; + + updateDataverseFeaturedItemsResponse = UtilIT.updateDataverseFeaturedItems(dataverseAlias, ids, contents, orders, keepFiles, pathsToFiles, apiToken); + updateDataverseFeaturedItemsResponse.then().assertThat() + .body("data.size()", equalTo(1)) + .body("data[0].content", equalTo("Content 1 updated")) + .body("data[0].imageFileName", equalTo(null)) + .body("data[0].imageFileUrl", equalTo(null)) + .body("data[0].displayOrder", equalTo(0)) .statusCode(OK.getStatusCode()); } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index d0da601eb9a..6e95dace0e3 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -4459,34 +4459,25 @@ static Response updateDataverseFeaturedItems( List pathsToFiles, String apiToken) { - RequestSpecification requestSpecification = given() + RequestSpecification requestSpec = given() .header(API_TOKEN_HTTP_HEADER, apiToken) .contentType(ContentType.MULTIPART); - int size = contents.size(); - if (ids.size() != size || orders.size() != size || keepFiles.size() != size || pathsToFiles.size() != size) { - throw new IllegalArgumentException("'ids', 'contents', 'orders', 'keepFiles' and 'pathsToFiles' lists must have the same size."); - } - - for (int i = 0; i < size; i++) { - Long id = ids.get(i); - String content = contents.get(i); - Integer order = orders.get(i); - boolean keepFile = keepFiles.get(i); + for (int i = 0; i < contents.size(); i++) { + requestSpec.multiPart("content", contents.get(i)) + .multiPart("displayOrder", orders.get(i)) + .multiPart("keepFile", keepFiles.get(i)) + .multiPart("id", ids.get(i)); - requestSpecification.multiPart("content", content); - requestSpecification.multiPart("displayOrder", order); - requestSpecification.multiPart("keepFile", keepFile); - requestSpecification.multiPart("id", id); - - String pathToFile = pathsToFiles.get(i); + String pathToFile = pathsToFiles != null ? pathsToFiles.get(i) : null; if (pathToFile != null && !pathToFile.isEmpty()) { - requestSpecification.multiPart("file", new File(pathToFile)); + requestSpec.multiPart("fileName", Paths.get(pathToFile).getFileName().toString()) + .multiPart("file", new File(pathToFile)); + } else { + requestSpec.multiPart("fileName", ""); } } - return requestSpecification - .when() - .put("/api/dataverses/" + dataverseAlias + "/featuredItems"); + return requestSpec.when().put("/api/dataverses/" + dataverseAlias + "/featuredItems"); } } From aabc4d3f18c6f47a8f7fdc05189b7c9e5e4b3af2 Mon Sep 17 00:00:00 2001 From: GPortas Date: Fri, 10 Jan 2025 10:08:00 +0000 Subject: [PATCH 082/137] Added: deleteDataverseFeaturedItems endpoint --- .../harvard/iq/dataverse/api/Dataverses.java | 14 ++++++++ src/main/java/propertyFiles/Bundle.properties | 1 + .../iq/dataverse/api/DataversesIT.java | 33 +++++++++++++++++++ .../edu/harvard/iq/dataverse/api/UtilIT.java | 6 ++++ 4 files changed, 54 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index d4bbb50b971..22eea188150 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -1,5 +1,6 @@ package edu.harvard.iq.dataverse.api; +import com.google.api.client.util.ArrayMap; import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.api.auth.AuthRequired; import edu.harvard.iq.dataverse.api.datadeposit.SwordServiceBean; @@ -1848,4 +1849,17 @@ public Response updateFeaturedItems( return wr.getResponse(); } } + + @DELETE + @AuthRequired + @Path("{identifier}/featuredItems") + public Response deleteFeaturedItems(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf) { + try { + Dataverse dataverse = findDataverseOrDie(dvIdtf); + execCommand(new UpdateDataverseFeaturedItemsCommand(createDataverseRequest(getRequestUser(crc)), dataverse, new ArrayList<>(), new ArrayMap<>())); + return ok(BundleUtil.getStringFromBundle("dataverse.delete.featuredItems.success")); + } catch (WrappedResponse e) { + return e.getResponse(); + } + } } diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 07962712008..88358af5468 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -991,6 +991,7 @@ dataverse.create.featuredItem.error.fileSizeExceedsLimit=File exceeds the maximu dataverse.create.featuredItem.error.invalidFileType=Invalid image file type dataverse.update.featuredItems.error.missingInputParams=All input parameters (id, content, displayOrder, keepFile, fileNames) are required. dataverse.update.featuredItems.error.inputListsSizeMismatch=All input lists (id, content, displayOrder, keepFile, fileNames) must have the same size. +dataverse.delete.featuredItems.success=All featured items of this Dataverse have been successfully deleted. # rolesAndPermissionsFragment.xhtml # advanced.xhtml diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index ea29fb487fc..c6e9a8a5975 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -1743,4 +1743,37 @@ public void testUpdateFeaturedItems() { .body("data[0].displayOrder", equalTo(0)) .statusCode(OK.getStatusCode()); } + + @Test + public void testDeleteFeaturedItems() { + Response createUserResponse = UtilIT.createRandomUser(); + String apiToken = UtilIT.getApiTokenFromResponse(createUserResponse); + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode()); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + // Create test featured items + + List ids = Arrays.asList(0L, 0L, 0L); + List contents = Arrays.asList("Content 1", "Content 2", "Content 3"); + List orders = Arrays.asList(0, 1, 2); + List keepFiles = Arrays.asList(false, false, false); + List pathsToFiles = Arrays.asList("src/test/resources/images/coffeeshop.png", null, null); + + Response updateDataverseFeaturedItemsResponse = UtilIT.updateDataverseFeaturedItems(dataverseAlias, ids, contents, orders, keepFiles, pathsToFiles, apiToken); + updateDataverseFeaturedItemsResponse.then().assertThat() + .body("data.size()", equalTo(3)) + .statusCode(OK.getStatusCode()); + + // Check that the featured items are successfully deleted when calling the delete endpoint + + Response deleteDataverseFeaturedItemsResponse = UtilIT.deleteDataverseFeaturedItems(dataverseAlias, apiToken); + deleteDataverseFeaturedItemsResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + + Response listFeaturedItemsResponse = UtilIT.listDataverseFeaturedItems(dataverseAlias, apiToken); + listFeaturedItemsResponse.then() + .body("data.size()", equalTo(0)) + .assertThat().statusCode(OK.getStatusCode()); + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 666f765e4a6..7e8da8a3310 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -4484,4 +4484,10 @@ static Response updateDataverseFeaturedItems( return requestSpec.when().put("/api/dataverses/" + dataverseAlias + "/featuredItems"); } + + static Response deleteDataverseFeaturedItems(String dataverseAlias, String apiToken) { + return given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .delete("/api/dataverses/" + dataverseAlias + "/featuredItems"); + } } From b2eca4bebd56c91480c1d653afaf69d539849ab5 Mon Sep 17 00:00:00 2001 From: GPortas Date: Fri, 10 Jan 2025 12:30:39 +0000 Subject: [PATCH 083/137] Added: featured item content validation --- .../iq/dataverse/DataverseFeaturedItem.java | 4 +- ...ractWriteDataverseFeaturedItemCommand.java | 20 ++++++++++ .../CreateDataverseFeaturedItemCommand.java | 5 ++- .../UpdateDataverseFeaturedItemCommand.java | 2 +- src/main/java/propertyFiles/Bundle.properties | 2 + ...reateDataverseFeaturedItemCommandTest.java | 40 ++++++++++++++++++- 6 files changed, 67 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java index fb2077c5f27..3aa15825ac6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java @@ -13,6 +13,8 @@ @Entity public class DataverseFeaturedItem { + public static final int MAX_FEATURED_ITEM_CONTENT_SIZE = 15000; + @Id @GeneratedValue(strategy = GenerationType.IDENTITY) private Long id; @@ -22,7 +24,7 @@ public class DataverseFeaturedItem { private Dataverse dataverse; @NotBlank - @Size(max = 15000) + @Size(max = MAX_FEATURED_ITEM_CONTENT_SIZE) @Lob @Column(columnDefinition = "TEXT", nullable = false) private String content; diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseFeaturedItemCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseFeaturedItemCommand.java index 0b3395d6f78..8d70ff5a6ab 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseFeaturedItemCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseFeaturedItemCommand.java @@ -12,6 +12,7 @@ import java.io.IOException; import java.io.InputStream; +import java.text.MessageFormat; import java.util.List; /** @@ -27,6 +28,25 @@ public AbstractWriteDataverseFeaturedItemCommand(DataverseRequest request, Datav this.dataverse = affectedDataverse; } + protected void validateAndSetContent(DataverseFeaturedItem featuredItem, String content) throws InvalidCommandArgumentsException { + if (content == null) { + throw new InvalidCommandArgumentsException( + BundleUtil.getStringFromBundle("dataverse.create.featuredItem.error.contentShouldBeProvided"), + this + ); + } + if (content.length() > DataverseFeaturedItem.MAX_FEATURED_ITEM_CONTENT_SIZE) { + throw new InvalidCommandArgumentsException( + MessageFormat.format( + BundleUtil.getStringFromBundle("dataverse.create.featuredItem.error.contentExceedsLengthLimit"), + List.of(DataverseFeaturedItem.MAX_FEATURED_ITEM_CONTENT_SIZE) + ), + this + ); + } + featuredItem.setContent(content); + } + protected void setFileImageIfAvailableOrNull(DataverseFeaturedItem featuredItem, String imageFileName, InputStream imageFileInputStream, CommandContext ctxt) throws CommandException { if (imageFileName != null && imageFileInputStream != null) { try { diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java index e1f21587bb0..b21ed1403cd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java @@ -25,6 +25,9 @@ public CreateDataverseFeaturedItemCommand(DataverseRequest request, public DataverseFeaturedItem execute(CommandContext ctxt) throws CommandException { DataverseFeaturedItem dataverseFeaturedItem = new DataverseFeaturedItem(); + validateAndSetContent(dataverseFeaturedItem, newDataverseFeaturedItemDTO.getContent()); + dataverseFeaturedItem.setDisplayOrder(newDataverseFeaturedItemDTO.getDisplayOrder()); + setFileImageIfAvailableOrNull( dataverseFeaturedItem, newDataverseFeaturedItemDTO.getImageFileName(), @@ -32,8 +35,6 @@ public DataverseFeaturedItem execute(CommandContext ctxt) throws CommandExceptio ctxt ); - dataverseFeaturedItem.setContent(newDataverseFeaturedItemDTO.getContent()); - dataverseFeaturedItem.setDisplayOrder(newDataverseFeaturedItemDTO.getDisplayOrder()); dataverseFeaturedItem.setDataverse(dataverse); return ctxt.dataverseFeaturedItems().save(dataverseFeaturedItem); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemCommand.java index 51e5d43a7aa..a54f0324a3e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemCommand.java @@ -24,7 +24,7 @@ public UpdateDataverseFeaturedItemCommand(DataverseRequest request, @Override public DataverseFeaturedItem execute(CommandContext ctxt) throws CommandException { - dataverseFeaturedItem.setContent(updatedDataverseFeaturedItemDTO.getContent()); + validateAndSetContent(dataverseFeaturedItem, updatedDataverseFeaturedItemDTO.getContent()); dataverseFeaturedItem.setDisplayOrder(updatedDataverseFeaturedItemDTO.getDisplayOrder()); if (!updatedDataverseFeaturedItemDTO.isKeepFile()) { diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 88358af5468..f3264dabd2b 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -989,6 +989,8 @@ dataverse.create.error.jsonparsetodataverse=Error parsing the POSTed json into a dataverse.create.featuredItem.error.imageFileProcessing=Error processing featured item file: {0} dataverse.create.featuredItem.error.fileSizeExceedsLimit=File exceeds the maximum size of {0} dataverse.create.featuredItem.error.invalidFileType=Invalid image file type +dataverse.create.featuredItem.error.contentShouldBeProvided=Featured item 'content' property should be provided. +dataverse.create.featuredItem.error.contentExceedsLengthLimit=Featured item content exceeds the maximum allowed length of {0} characters. dataverse.update.featuredItems.error.missingInputParams=All input parameters (id, content, displayOrder, keepFile, fileNames) are required. dataverse.update.featuredItems.error.inputListsSizeMismatch=All input lists (id, content, displayOrder, keepFile, fileNames) must have the same size. dataverse.delete.featuredItems.success=All featured items of this Dataverse have been successfully deleted. diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommandTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommandTest.java index d8bb6549c60..d584b1795a9 100644 --- a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommandTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommandTest.java @@ -7,6 +7,7 @@ import edu.harvard.iq.dataverse.engine.command.CommandContext; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.InvalidCommandArgumentsException; +import edu.harvard.iq.dataverse.util.BundleUtil; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.InjectMocks; @@ -15,6 +16,8 @@ import java.io.IOException; import java.io.InputStream; +import java.text.MessageFormat; +import java.util.List; import static edu.harvard.iq.dataverse.mocks.MocksFactory.makeRequest; import static org.junit.jupiter.api.Assertions.*; @@ -106,8 +109,8 @@ void execute_imageFileProcessingFails_throwsCommandException() throws IOExceptio } @Test - void execute_invalidArgumentsProvided_throwsInvalidCommandArgumentsException() throws IOException, DataverseFeaturedItemServiceBean.InvalidImageFileException { - testNewDataverseFeaturedItemDTO.setImageFileName("invalid.png"); + void execute_invalidFileTypeProvided_throwsInvalidCommandArgumentsException() throws IOException, DataverseFeaturedItemServiceBean.InvalidImageFileException { + testNewDataverseFeaturedItemDTO.setImageFileName("invalid.type"); InputStream inputStreamMock = mock(InputStream.class); testNewDataverseFeaturedItemDTO.setImageFileInputStream(inputStreamMock); @@ -117,4 +120,37 @@ void execute_invalidArgumentsProvided_throwsInvalidCommandArgumentsException() t InvalidCommandArgumentsException exception = assertThrows(InvalidCommandArgumentsException.class, () -> sut.execute(contextStub)); assertTrue(exception.getMessage().contains("Invalid file type")); } + + @Test + void execute_contentNotProvided_throwsInvalidCommandArgumentsException() { + testNewDataverseFeaturedItemDTO.setContent(null); + InputStream inputStreamMock = mock(InputStream.class); + testNewDataverseFeaturedItemDTO.setImageFileInputStream(inputStreamMock); + + InvalidCommandArgumentsException exception = assertThrows(InvalidCommandArgumentsException.class, () -> sut.execute(contextStub)); + assertEquals( + BundleUtil.getStringFromBundle("dataverse.create.featuredItem.error.contentShouldBeProvided"), + exception.getMessage() + ); + } + + @Test + void execute_contentExceedsLimit_throwsInvalidCommandArgumentsException() { + testNewDataverseFeaturedItemDTO.setContent(createContentExceedingMaxLength()); + InputStream inputStreamMock = mock(InputStream.class); + testNewDataverseFeaturedItemDTO.setImageFileInputStream(inputStreamMock); + + InvalidCommandArgumentsException exception = assertThrows(InvalidCommandArgumentsException.class, () -> sut.execute(contextStub)); + assertEquals( + MessageFormat.format( + BundleUtil.getStringFromBundle("dataverse.create.featuredItem.error.contentExceedsLengthLimit"), + List.of(DataverseFeaturedItem.MAX_FEATURED_ITEM_CONTENT_SIZE) + ), + exception.getMessage() + ); + } + + private String createContentExceedingMaxLength() { + return "a".repeat(Math.max(0, DataverseFeaturedItem.MAX_FEATURED_ITEM_CONTENT_SIZE + 1)); + } } From a2c63e226a064a5481f5b8ec7007431910cb9c76 Mon Sep 17 00:00:00 2001 From: stevenferey Date: Fri, 10 Jan 2025 16:25:31 +0100 Subject: [PATCH 084/137] Adaptations of language keys --- src/main/java/propertyFiles/Bundle.properties | 16 ++++++++-------- src/main/webapp/dashboard-movedataverse.xhtml | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 7a05abeb311..a87bbc760fc 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -760,12 +760,12 @@ dashboard.list_users.api.auth.not_superuser=Forbidden. You must be a superuser. #dashboard-movedataset.xhtml dashboard.move.dataset.header=Dashboard - Move Data dashboard.move.dataset.message=Manage and curate your installation by moving datasets from one host dataverse to another. See also Managing Datasets and Dataverses in the Admin Guide. -dashboard.move.dataset.selectdataset.header=Dataset to Move -dashboard.move.dataset.newdataverse.header=New Host Dataverse +dashboard.move.dataset.selectdataset.header=Dataset to move +dashboard.move.dataset.newdataverse.header=New dataverse collection host dashboard.move.dataset.dataset.label=Dataset dashboard.move.dataset.dataverse.label=Dataverse -dashboard.move.dataset.confirm.dialog=Are you sure want to move this dataset? -dashboard.move.dataset.confirm.yes=Yes, Move Dataset +dashboard.move.dataset.confirm.dialog=Are you sure you want to move this dataset? +dashboard.move.dataset.confirm.yes=Yes, move this dataset dashboard.move.dataset.message.success=The dataset "{0}" ({1}) has been successfully moved to {2}. dashboard.move.dataset.message.failure.summary=Failed to moved dataset dashboard.move.dataset.message.failure.details=The dataset "{0}" ({1}) could not be moved to {2}. {3}{4} @@ -785,11 +785,11 @@ dashboard.move.dataset.command.error.unforced.suggestForce=Forcing this move is dashboard.move.dataverse.header=Dashboard - Move Data dashboard.move.dataverse.message.summary=Move Dataverse Collection dashboard.move.dataverse.message.detail=Manage and curate your installation by moving a dataverse collection from one host dataverse collection to another. See also Managing Datasets and Dataverses in the Admin Guide. -dashboard.move.dataverse.selectdataverse.header=Dataverse to Move -dashboard.move.dataverse.newdataverse.header=New Host Dataverse +dashboard.move.dataverse.selectdataverse.header=Dataverse collection to move +dashboard.move.dataverse.newdataverse.header=New dataverse collection host dashboard.move.dataverse.label=Dataverse -dashboard.move.dataverse.confirm.dialog=Are you sure want to move this dataverse? -dashboard.move.dataverse.confirm.yes=Yes, Move Dataverse +dashboard.move.dataverse.confirm.dialog=Are you sure you want to move this dataverse collection? +dashboard.move.dataverse.confirm.yes=Yes, move this collection dashboard.move.dataverse.message.success=The dataverse "{0}" has been successfully moved to {1}. dashboard.move.dataverse.message.failure.summary=Failed to moved dataverse dashboard.move.dataverse.message.failure.details=The dataverse "{0}" could not be moved to {1}. {2} diff --git a/src/main/webapp/dashboard-movedataverse.xhtml b/src/main/webapp/dashboard-movedataverse.xhtml index 81a5d774994..ef54e21145d 100644 --- a/src/main/webapp/dashboard-movedataverse.xhtml +++ b/src/main/webapp/dashboard-movedataverse.xhtml @@ -16,7 +16,7 @@ - + @@ -120,7 +120,7 @@
- +

#{bundle['dashboard.move.dataverse.confirm.dialog']}

Date: Sat, 11 Jan 2025 12:51:59 +0000 Subject: [PATCH 085/137] Added: order featured items results by display order and missing tests added --- .../iq/dataverse/DataverseFeaturedItem.java | 5 +- .../DataverseFeaturedItemServiceBean.java | 7 + .../ListDataverseFeaturedItemsCommand.java | 7 +- .../UpdateDataverseFeaturedItemsCommand.java | 2 +- .../iq/dataverse/api/DataversesIT.java | 123 +++++++++++++++--- 5 files changed, 116 insertions(+), 28 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java index 3aa15825ac6..ee91c258426 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java @@ -8,9 +8,12 @@ @NamedQueries({ @NamedQuery(name = "DataverseFeaturedItem.deleteById", - query = "DELETE FROM DataverseFeaturedItem item WHERE item.id=:id") + query = "DELETE FROM DataverseFeaturedItem item WHERE item.id=:id"), + @NamedQuery(name = "DataverseFeaturedItem.findByDataverseOrderedByDisplayOrder", + query = "SELECT item FROM DataverseFeaturedItem item WHERE item.dataverse = :dataverse ORDER BY item.displayOrder ASC") }) @Entity +@Table(indexes = @Index(columnList = "displayOrder")) public class DataverseFeaturedItem { public static final int MAX_FEATURED_ITEM_CONTENT_SIZE = 15000; diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java index 73f7b5c5057..89a970fae64 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java @@ -50,6 +50,13 @@ public void delete(Long id) { .executeUpdate(); } + public List findAllByDataverseOrdered(Dataverse dataverse) { + return em + .createNamedQuery("DataverseFeaturedItem.findByDataverseOrderedByDisplayOrder", DataverseFeaturedItem.class) + .setParameter("dataverse", dataverse) + .getResultList(); + } + public InputStream getImageFileAsInputStream(DataverseFeaturedItem dataverseFeaturedItem) throws IOException { Path imagePath = Path.of(JvmSettings.DOCROOT_DIRECTORY.lookup(), JvmSettings.FEATURED_ITEMS_IMAGE_UPLOADS_DIRECTORY.lookup(), diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListDataverseFeaturedItemsCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListDataverseFeaturedItemsCommand.java index 9ff2d2e1e71..474747ccb0d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListDataverseFeaturedItemsCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListDataverseFeaturedItemsCommand.java @@ -8,10 +8,7 @@ import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; /** * Lists the featured items {@link DataverseFeaturedItem} of a {@link Dataverse}. @@ -27,7 +24,7 @@ public ListDataverseFeaturedItemsCommand(DataverseRequest request, Dataverse dat @Override public List execute(CommandContext ctxt) throws CommandException { - return dataverse.getDataverseFeaturedItems(); + return ctxt.dataverseFeaturedItems().findAllByDataverseOrdered(dataverse); } @Override diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemsCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemsCommand.java index 8fb5f4f09c8..12a972f2e47 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemsCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemsCommand.java @@ -38,7 +38,7 @@ public UpdateDataverseFeaturedItemsCommand(DataverseRequest request, Dataverse d public List execute(CommandContext ctxt) throws CommandException { List dataverseFeaturedItems = updateOrDeleteExistingFeaturedItems(ctxt); dataverseFeaturedItems.addAll(createNewFeaturedItems(ctxt)); - dataverseFeaturedItems.sort(Comparator.comparingLong(DataverseFeaturedItem::getId)); + dataverseFeaturedItems.sort(Comparator.comparingInt(DataverseFeaturedItem::getDisplayOrder)); return dataverseFeaturedItems; } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index c6e9a8a5975..ddd85f65562 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -1586,6 +1586,7 @@ public void testCreateFeaturedItem() { String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); // Should not return any error when not passing a file + Response createFeatureItemResponse = UtilIT.createDataverseFeaturedItem(dataverseAlias, apiToken, "test", 0, null); createFeatureItemResponse.then().assertThat() .statusCode(OK.getStatusCode()) @@ -1594,6 +1595,7 @@ public void testCreateFeaturedItem() { .body("data.displayOrder", equalTo(0)); // Should not return any error when passing correct file and data + String pathToTestFile = "src/test/resources/images/coffeeshop.png"; createFeatureItemResponse = UtilIT.createDataverseFeaturedItem(dataverseAlias, apiToken, "test", 1, pathToTestFile); createFeatureItemResponse.then().assertThat() @@ -1603,6 +1605,7 @@ public void testCreateFeaturedItem() { .statusCode(OK.getStatusCode()); // Should return bad request error when passing incorrect file type + pathToTestFile = "src/test/resources/tab/test.tab"; createFeatureItemResponse = UtilIT.createDataverseFeaturedItem(dataverseAlias, apiToken, "test", 0, pathToTestFile); createFeatureItemResponse.then().assertThat() @@ -1610,18 +1613,68 @@ public void testCreateFeaturedItem() { .statusCode(BAD_REQUEST.getStatusCode()); // Should return unauthorized error when user has no permissions + Response createRandomUser = UtilIT.createRandomUser(); String randomUserApiToken = UtilIT.getApiTokenFromResponse(createRandomUser); createFeatureItemResponse = UtilIT.createDataverseFeaturedItem(dataverseAlias, randomUserApiToken, "test", 0, pathToTestFile); createFeatureItemResponse.then().assertThat().statusCode(UNAUTHORIZED.getStatusCode()); // Should return not found error when dataverse does not exist + createFeatureItemResponse = UtilIT.createDataverseFeaturedItem("thisDataverseDoesNotExist", apiToken, "test", 0, pathToTestFile); createFeatureItemResponse.then().assertThat() .body("message", equalTo("Can't find dataverse with identifier='thisDataverseDoesNotExist'")) .statusCode(NOT_FOUND.getStatusCode()); } + @Test + public void testListFeaturedItems() { + Response createUserResponse = UtilIT.createRandomUser(); + String apiToken = UtilIT.getApiTokenFromResponse(createUserResponse); + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode()); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + // Create test items + + List ids = Arrays.asList(0L, 0L, 0L); + List contents = Arrays.asList("Content 1", "Content 2", "Content 3"); + List orders = Arrays.asList(2, 1, 0); + List keepFiles = Arrays.asList(false, false, false); + List pathsToFiles = Arrays.asList("src/test/resources/images/coffeeshop.png", null, null); + + Response updateDataverseFeaturedItemsResponse = UtilIT.updateDataverseFeaturedItems(dataverseAlias, ids, contents, orders, keepFiles, pathsToFiles, apiToken); + updateDataverseFeaturedItemsResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + + // Items should be retrieved with all their properties and sorted by displayOrder + + Response listDataverseFeaturedItemsResponse = UtilIT.listDataverseFeaturedItems(dataverseAlias, apiToken); + listDataverseFeaturedItemsResponse.then().assertThat() + .body("data.size()", equalTo(3)) + .body("data[0].content", equalTo("Content 3")) + .body("data[0].imageFileName", equalTo(null)) + .body("data[0].imageFileUrl", equalTo(null)) + .body("data[0].displayOrder", equalTo(0)) + .body("data[1].content", equalTo("Content 2")) + .body("data[1].imageFileName", equalTo(null)) + .body("data[1].imageFileUrl", equalTo(null)) + .body("data[1].displayOrder", equalTo(1)) + .body("data[2].content", equalTo("Content 1")) + .body("data[2].imageFileName", equalTo("coffeeshop.png")) + .body("data[2].imageFileUrl", containsString(UtilIT.getRestAssuredBaseUri() + "/api/access/dataverseFeatureItemImage/")) + .body("data[2].displayOrder", equalTo(2)) + .statusCode(OK.getStatusCode()); + + // Should return not found error when dataverse does not exist + + listDataverseFeaturedItemsResponse = UtilIT.listDataverseFeaturedItems("thisDataverseDoesNotExist", apiToken); + listDataverseFeaturedItemsResponse.then().assertThat() + .body("message", equalTo("Can't find dataverse with identifier='thisDataverseDoesNotExist'")) + .statusCode(NOT_FOUND.getStatusCode()); + + } + @Test public void testUpdateFeaturedItems() { Response createUserResponse = UtilIT.createRandomUser(); @@ -1671,22 +1724,22 @@ public void testUpdateFeaturedItems() { updateDataverseFeaturedItemsResponse = UtilIT.updateDataverseFeaturedItems(dataverseAlias, ids, contents, orders, keepFiles, pathsToFiles, apiToken); updateDataverseFeaturedItemsResponse.then().assertThat() .body("data.size()", equalTo(3)) - .body("data[0].content", equalTo("Content 1 updated")) - .body("data[0].imageFileName", equalTo("coffeeshop.png")) - .body("data[0].imageFileUrl", containsString(baseUri + "/api/access/dataverseFeatureItemImage/")) - .body("data[0].displayOrder", equalTo(1)) - .body("data[1].content", equalTo("Content 2")) - .body("data[1].imageFileName", equalTo(null)) - .body("data[1].imageFileUrl", equalTo(null)) - .body("data[1].displayOrder", equalTo(0)) + .body("data[0].content", equalTo("Content 2")) + .body("data[0].imageFileName", equalTo(null)) + .body("data[0].imageFileUrl", equalTo(null)) + .body("data[0].displayOrder", equalTo(0)) + .body("data[1].content", equalTo("Content 1 updated")) + .body("data[1].imageFileName", equalTo("coffeeshop.png")) + .body("data[1].imageFileUrl", containsString(baseUri + "/api/access/dataverseFeatureItemImage/")) + .body("data[1].displayOrder", equalTo(1)) .body("data[2].content", equalTo("Content 3")) .body("data[2].imageFileName", equalTo(null)) .body("data[2].imageFileUrl", equalTo(null)) .body("data[2].displayOrder", equalTo(2)) .statusCode(OK.getStatusCode()); - Long firstItemIdAfterUpdate = JsonPath.from(updateDataverseFeaturedItemsResponse.body().asString()).getLong("data[0].id"); - Long secondItemIdAfterUpdate = JsonPath.from(updateDataverseFeaturedItemsResponse.body().asString()).getLong("data[1].id"); + Long firstItemIdAfterUpdate = JsonPath.from(updateDataverseFeaturedItemsResponse.body().asString()).getLong("data[1].id"); + Long secondItemIdAfterUpdate = JsonPath.from(updateDataverseFeaturedItemsResponse.body().asString()).getLong("data[0].id"); Long thirdItemIdAfterUpdate = JsonPath.from(updateDataverseFeaturedItemsResponse.body().asString()).getLong("data[2].id"); assertEquals(firstItemId, firstItemIdAfterUpdate); @@ -1704,25 +1757,25 @@ public void testUpdateFeaturedItems() { updateDataverseFeaturedItemsResponse = UtilIT.updateDataverseFeaturedItems(dataverseAlias, ids, contents, orders, keepFiles, pathsToFiles, apiToken); updateDataverseFeaturedItemsResponse.then().assertThat() .body("data.size()", equalTo(3)) - .body("data[0].content", equalTo("Content 1 updated")) - .body("data[0].imageFileName", equalTo(null)) - .body("data[0].imageFileUrl", equalTo(null)) - .body("data[0].displayOrder", equalTo(1)) - .body("data[1].content", equalTo("Content 2")) - .body("data[1].imageFileName", equalTo("coffeeshop.png")) - .body("data[1].imageFileUrl", containsString(baseUri + "/api/access/dataverseFeatureItemImage/")) - .body("data[1].displayOrder", equalTo(0)) + .body("data[0].content", equalTo("Content 2")) + .body("data[0].imageFileName", equalTo("coffeeshop.png")) + .body("data[0].imageFileUrl", containsString(baseUri + "/api/access/dataverseFeatureItemImage/")) + .body("data[0].displayOrder", equalTo(0)) + .body("data[1].content", equalTo("Content 1 updated")) + .body("data[1].imageFileName", equalTo(null)) + .body("data[1].imageFileUrl", equalTo(null)) + .body("data[1].displayOrder", equalTo(1)) .body("data[2].content", equalTo("Content 3")) .body("data[2].imageFileName", equalTo(null)) .body("data[2].imageFileUrl", equalTo(null)) .body("data[2].displayOrder", equalTo(2)) .statusCode(OK.getStatusCode()); - Long firstItemIdAftersSecondUpdate = JsonPath.from(updateDataverseFeaturedItemsResponse.body().asString()).getLong("data[0].id"); - Long secondItemIdAfterSecondUpdate = JsonPath.from(updateDataverseFeaturedItemsResponse.body().asString()).getLong("data[1].id"); + Long firstItemIdAfterSecondUpdate = JsonPath.from(updateDataverseFeaturedItemsResponse.body().asString()).getLong("data[1].id"); + Long secondItemIdAfterSecondUpdate = JsonPath.from(updateDataverseFeaturedItemsResponse.body().asString()).getLong("data[0].id"); Long thirdItemIdAfterSecondUpdate = JsonPath.from(updateDataverseFeaturedItemsResponse.body().asString()).getLong("data[2].id"); - assertEquals(firstItemId, firstItemIdAftersSecondUpdate); + assertEquals(firstItemId, firstItemIdAfterSecondUpdate); assertEquals(secondItemIdAfterUpdate, secondItemIdAfterSecondUpdate); assertNotEquals(thirdItemIdAfterUpdate, thirdItemIdAfterSecondUpdate); @@ -1742,6 +1795,20 @@ public void testUpdateFeaturedItems() { .body("data[0].imageFileUrl", equalTo(null)) .body("data[0].displayOrder", equalTo(0)) .statusCode(OK.getStatusCode()); + + // Should return unauthorized error when user has no permissions + + Response createRandomUser = UtilIT.createRandomUser(); + String randomUserApiToken = UtilIT.getApiTokenFromResponse(createRandomUser); + updateDataverseFeaturedItemsResponse = UtilIT.updateDataverseFeaturedItems(dataverseAlias, ids, contents, orders, keepFiles, pathsToFiles, randomUserApiToken); + updateDataverseFeaturedItemsResponse.then().assertThat().statusCode(UNAUTHORIZED.getStatusCode()); + + // Should return not found error when dataverse does not exist + + updateDataverseFeaturedItemsResponse = UtilIT.updateDataverseFeaturedItems("thisDataverseDoesNotExist", ids, contents, orders, keepFiles, pathsToFiles, apiToken); + updateDataverseFeaturedItemsResponse.then().assertThat() + .body("message", equalTo("Can't find dataverse with identifier='thisDataverseDoesNotExist'")) + .statusCode(NOT_FOUND.getStatusCode()); } @Test @@ -1775,5 +1842,19 @@ public void testDeleteFeaturedItems() { listFeaturedItemsResponse.then() .body("data.size()", equalTo(0)) .assertThat().statusCode(OK.getStatusCode()); + + // Should return unauthorized error when user has no permissions + + Response createRandomUser = UtilIT.createRandomUser(); + String randomUserApiToken = UtilIT.getApiTokenFromResponse(createRandomUser); + deleteDataverseFeaturedItemsResponse = UtilIT.deleteDataverseFeaturedItems(dataverseAlias, randomUserApiToken); + deleteDataverseFeaturedItemsResponse.then().assertThat().statusCode(UNAUTHORIZED.getStatusCode()); + + // Should return not found error when dataverse does not exist + + deleteDataverseFeaturedItemsResponse = UtilIT.deleteDataverseFeaturedItems("thisDataverseDoesNotExist", apiToken); + deleteDataverseFeaturedItemsResponse.then().assertThat() + .body("message", equalTo("Can't find dataverse with identifier='thisDataverseDoesNotExist'")) + .statusCode(NOT_FOUND.getStatusCode()); } } From e265ae873b0c053e4b6b0b3542d8f4a8323bcda9 Mon Sep 17 00:00:00 2001 From: GPortas Date: Mon, 13 Jan 2025 14:12:36 +0000 Subject: [PATCH 086/137] Fixed: response message string formatting --- src/main/java/propertyFiles/Bundle.properties | 4 ++-- .../dataverse/api/DataverseFeaturedItemsIT.java | 15 ++++++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index f3264dabd2b..9a89fc0eee5 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -3126,5 +3126,5 @@ authenticationServiceBean.errors.invalidBearerToken=Could not parse bearer token authenticationServiceBean.errors.bearerTokenDetectedNoOIDCProviderConfigured=Bearer token detected, no OIDC provider configured. #DataverseFeaturedItems.java -dataverseFeaturedItems.errors.notFound=Can't find dataverse featured item with identifier='{0}' -dataverseFeaturedItems.delete.successful=Successfully deleted dataverse featured item with identifier='{0}' +dataverseFeaturedItems.errors.notFound=Can't find dataverse featured item with identifier {0} +dataverseFeaturedItems.delete.successful=Successfully deleted dataverse featured item with identifier {0} diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java index d53414d4207..edb1e2bebba 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java @@ -1,11 +1,14 @@ package edu.harvard.iq.dataverse.api; +import edu.harvard.iq.dataverse.util.BundleUtil; import io.restassured.RestAssured; import io.restassured.path.json.JsonPath; import io.restassured.response.Response; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import java.text.MessageFormat; + import static jakarta.ws.rs.core.Response.Status.*; import static org.hamcrest.CoreMatchers.equalTo; @@ -24,7 +27,9 @@ public void testDeleteFeaturedItem() { // Should return not found when passing incorrect item id Response deleteFeatureItemResponse = UtilIT.deleteDataverseFeaturedItem(100000L, apiToken); - deleteFeatureItemResponse.then().assertThat().statusCode(NOT_FOUND.getStatusCode()); + deleteFeatureItemResponse.then() + .body("message", equalTo(MessageFormat.format(BundleUtil.getStringFromBundle("dataverseFeaturedItems.errors.notFound"), featuredItemId))) + .assertThat().statusCode(NOT_FOUND.getStatusCode()); // Should return unauthorized when passing correct id and user does not have permissions String randomUserApiToken = createUserAndGetApiToken(); @@ -33,7 +38,9 @@ public void testDeleteFeaturedItem() { // Should delete featured item when passing correct id and user has permissions deleteFeatureItemResponse = UtilIT.deleteDataverseFeaturedItem(featuredItemId, apiToken); - deleteFeatureItemResponse.then().assertThat().statusCode(OK.getStatusCode()); + deleteFeatureItemResponse.then() + .body("data.message", equalTo(MessageFormat.format(BundleUtil.getStringFromBundle("dataverseFeaturedItems.delete.successful"), featuredItemId))) + .assertThat().statusCode(OK.getStatusCode()); Response listFeaturedItemsResponse = UtilIT.listDataverseFeaturedItems(dataverseAlias, apiToken); listFeaturedItemsResponse.then() @@ -49,7 +56,9 @@ public void testUpdateFeaturedItem() { // Should return not found when passing incorrect item id Response updateFeatureItemResponse = UtilIT.updateDataverseFeaturedItem(100000L, "updatedTitle", 1, false, null, apiToken); - updateFeatureItemResponse.then().assertThat().statusCode(NOT_FOUND.getStatusCode()); + updateFeatureItemResponse.then() + .body("message", equalTo(MessageFormat.format(BundleUtil.getStringFromBundle("dataverseFeaturedItems.errors.notFound"), featuredItemId))) + .assertThat().statusCode(NOT_FOUND.getStatusCode()); // Should return unauthorized when passing correct id and user does not have permissions String randomUserApiToken = createUserAndGetApiToken(); From bff8e94dd57d62be9e61139d7877b942b1b4633a Mon Sep 17 00:00:00 2001 From: GPortas Date: Mon, 13 Jan 2025 14:40:59 +0000 Subject: [PATCH 087/137] Added: featured items deletion in DeleteDataverseCommand --- src/main/java/edu/harvard/iq/dataverse/Dataverse.java | 4 ++++ .../engine/command/impl/DeleteDataverseCommand.java | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java index 024d37a730d..eaebf654dca 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java +++ b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java @@ -358,6 +358,10 @@ public List getDataverseFeaturedItems() { return this.dataverseFeaturedItems; } + public void setDataverseFeaturedItems(List dataverseFeaturedItems) { + this.dataverseFeaturedItems = dataverseFeaturedItems; + } + public List getParentGuestbooks() { List retList = new ArrayList<>(); Dataverse testDV = this; diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataverseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataverseCommand.java index c7c592f9458..7fbb659ab96 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataverseCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataverseCommand.java @@ -1,6 +1,7 @@ package edu.harvard.iq.dataverse.engine.command.impl; import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.DataverseFeaturedItem; import edu.harvard.iq.dataverse.DataverseFieldTypeInputLevel; import edu.harvard.iq.dataverse.authorization.DataverseRole; import edu.harvard.iq.dataverse.RoleAssignment; @@ -78,6 +79,14 @@ protected void executeImpl(CommandContext ctxt) throws CommandException { ctxt.em().remove(merged); } doomed.setDataverseFieldTypeInputLevels(new ArrayList<>()); + + // Featured Items + for (DataverseFeaturedItem featuredItem : doomed.getDataverseFeaturedItems()) { + DataverseFeaturedItem merged = ctxt.em().merge(featuredItem); + ctxt.em().remove(merged); + } + doomed.setDataverseFeaturedItems(new ArrayList<>()); + // DATAVERSE Dataverse doomedAndMerged = ctxt.em().merge(doomed); ctxt.em().remove(doomedAndMerged); From 5c61511d465baec72395e52cd6e35b347f6fb506 Mon Sep 17 00:00:00 2001 From: GPortas Date: Mon, 13 Jan 2025 15:27:22 +0000 Subject: [PATCH 088/137] Added: sanitizing featured item content property --- .../impl/AbstractWriteDataverseFeaturedItemCommand.java | 2 ++ .../iq/dataverse/api/DataverseFeaturedItemsIT.java | 8 ++++++-- .../edu/harvard/iq/dataverse/util/MarkupCheckerTest.java | 2 ++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseFeaturedItemCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseFeaturedItemCommand.java index 8d70ff5a6ab..ad7a8ca7807 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseFeaturedItemCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseFeaturedItemCommand.java @@ -9,6 +9,7 @@ import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.InvalidCommandArgumentsException; import edu.harvard.iq.dataverse.util.BundleUtil; +import edu.harvard.iq.dataverse.util.MarkupChecker; import java.io.IOException; import java.io.InputStream; @@ -35,6 +36,7 @@ protected void validateAndSetContent(DataverseFeaturedItem featuredItem, String this ); } + content = MarkupChecker.sanitizeBasicHTML(content); if (content.length() > DataverseFeaturedItem.MAX_FEATURED_ITEM_CONTENT_SIZE) { throw new InvalidCommandArgumentsException( MessageFormat.format( diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java index edb1e2bebba..d815fc77919 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java @@ -76,6 +76,10 @@ public void testUpdateFeaturedItem() { // Update featured item: set new image file updateFeatureItemResponse = UtilIT.updateDataverseFeaturedItem(featuredItemId, "updatedTitle1", 2, false, "src/test/resources/images/coffeeshop.png", apiToken); verifyUpdatedFeaturedItem(updateFeatureItemResponse, "updatedTitle1", "coffeeshop.png", 2); + + // Update featured item: set malicious content which should be sanitized + updateFeatureItemResponse = UtilIT.updateDataverseFeaturedItem(featuredItemId, "

hello

", 2, false, "src/test/resources/images/coffeeshop.png", apiToken); + verifyUpdatedFeaturedItem(updateFeatureItemResponse, "

hello

", "coffeeshop.png", 2); } private String createUserAndGetApiToken() { @@ -96,9 +100,9 @@ private Long createFeaturedItemAndGetId(String dataverseAlias, String apiToken, return createdFeaturedItem.getLong("data.id"); } - private void verifyUpdatedFeaturedItem(Response response, String expectedTitle, String expectedImageFileName, int expectedDisplayOrder) { + private void verifyUpdatedFeaturedItem(Response response, String expectedContent, String expectedImageFileName, int expectedDisplayOrder) { response.then().assertThat() - .body("data.content", equalTo(expectedTitle)) + .body("data.content", equalTo(expectedContent)) .body("data.imageFileName", equalTo(expectedImageFileName)) .body("data.displayOrder", equalTo(expectedDisplayOrder)) .statusCode(OK.getStatusCode()); diff --git a/src/test/java/edu/harvard/iq/dataverse/util/MarkupCheckerTest.java b/src/test/java/edu/harvard/iq/dataverse/util/MarkupCheckerTest.java index 07876e56eb8..19d656b2736 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/MarkupCheckerTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/MarkupCheckerTest.java @@ -23,6 +23,8 @@ public class MarkupCheckerTest { "'

hello

', '

hello

'", "'the Dataverse project in a new window', 'the Dataverse project in a new window'", "'the Dataverse project in a new window', 'the Dataverse project in a new window'", + // make sure we keep text as it is when it is not html + "'Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.', 'Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.'", "NULL, NULL" }, nullValues = {"NULL"}) public void testSanitizeBasicHTML(String unsafe, String safe) { From f4b6e0caba1a3626c2883f5df6a9f597a78fd426 Mon Sep 17 00:00:00 2001 From: GPortas Date: Tue, 14 Jan 2025 09:36:31 +0000 Subject: [PATCH 089/137] Added: empty content validation for featured items --- ...ractWriteDataverseFeaturedItemCommand.java | 2 +- src/main/java/propertyFiles/Bundle.properties | 2 +- ...reateDataverseFeaturedItemCommandTest.java | 30 +++++++++++++------ 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseFeaturedItemCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseFeaturedItemCommand.java index ad7a8ca7807..e76553dd9d4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseFeaturedItemCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseFeaturedItemCommand.java @@ -30,7 +30,7 @@ public AbstractWriteDataverseFeaturedItemCommand(DataverseRequest request, Datav } protected void validateAndSetContent(DataverseFeaturedItem featuredItem, String content) throws InvalidCommandArgumentsException { - if (content == null) { + if (content == null || content.trim().isEmpty()) { throw new InvalidCommandArgumentsException( BundleUtil.getStringFromBundle("dataverse.create.featuredItem.error.contentShouldBeProvided"), this diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 9a89fc0eee5..ea4bcbbd30f 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -989,7 +989,7 @@ dataverse.create.error.jsonparsetodataverse=Error parsing the POSTed json into a dataverse.create.featuredItem.error.imageFileProcessing=Error processing featured item file: {0} dataverse.create.featuredItem.error.fileSizeExceedsLimit=File exceeds the maximum size of {0} dataverse.create.featuredItem.error.invalidFileType=Invalid image file type -dataverse.create.featuredItem.error.contentShouldBeProvided=Featured item 'content' property should be provided. +dataverse.create.featuredItem.error.contentShouldBeProvided=Featured item 'content' property should be provided and not empty. dataverse.create.featuredItem.error.contentExceedsLengthLimit=Featured item content exceeds the maximum allowed length of {0} characters. dataverse.update.featuredItems.error.missingInputParams=All input parameters (id, content, displayOrder, keepFile, fileNames) are required. dataverse.update.featuredItems.error.inputListsSizeMismatch=All input lists (id, content, displayOrder, keepFile, fileNames) must have the same size. diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommandTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommandTest.java index d584b1795a9..01c5f499b0f 100644 --- a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommandTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommandTest.java @@ -10,6 +10,8 @@ import edu.harvard.iq.dataverse.util.BundleUtil; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.MockitoAnnotations; @@ -122,16 +124,14 @@ void execute_invalidFileTypeProvided_throwsInvalidCommandArgumentsException() th } @Test - void execute_contentNotProvided_throwsInvalidCommandArgumentsException() { - testNewDataverseFeaturedItemDTO.setContent(null); - InputStream inputStreamMock = mock(InputStream.class); - testNewDataverseFeaturedItemDTO.setImageFileInputStream(inputStreamMock); + void execute_contentIsNull_throwsInvalidCommandArgumentsException() { + assertContentShouldBeProvidedInvalidCommandArgumentsException(null); + } - InvalidCommandArgumentsException exception = assertThrows(InvalidCommandArgumentsException.class, () -> sut.execute(contextStub)); - assertEquals( - BundleUtil.getStringFromBundle("dataverse.create.featuredItem.error.contentShouldBeProvided"), - exception.getMessage() - ); + @ParameterizedTest + @ValueSource(strings = {" ", ""}) + void execute_contentIsEmpty_throwsInvalidCommandArgumentsException(String content) { + assertContentShouldBeProvidedInvalidCommandArgumentsException(content); } @Test @@ -150,6 +150,18 @@ void execute_contentExceedsLimit_throwsInvalidCommandArgumentsException() { ); } + private void assertContentShouldBeProvidedInvalidCommandArgumentsException(String content) { + testNewDataverseFeaturedItemDTO.setContent(content); + InputStream inputStreamMock = mock(InputStream.class); + testNewDataverseFeaturedItemDTO.setImageFileInputStream(inputStreamMock); + + InvalidCommandArgumentsException exception = assertThrows(InvalidCommandArgumentsException.class, () -> sut.execute(contextStub)); + assertEquals( + BundleUtil.getStringFromBundle("dataverse.create.featuredItem.error.contentShouldBeProvided"), + exception.getMessage() + ); + } + private String createContentExceedingMaxLength() { return "a".repeat(Math.max(0, DataverseFeaturedItem.MAX_FEATURED_ITEM_CONTENT_SIZE + 1)); } From 107a16e1efdfcfc130be4607dec56c5aa081fb59 Mon Sep 17 00:00:00 2001 From: GPortas Date: Tue, 14 Jan 2025 11:09:09 +0000 Subject: [PATCH 090/137] Changed: using advanced HTML sanitization for featured item content --- ...ractWriteDataverseFeaturedItemCommand.java | 2 +- .../iq/dataverse/util/MarkupChecker.java | 108 ++++++++++++------ .../api/DataverseFeaturedItemsIT.java | 6 +- .../iq/dataverse/util/MarkupCheckerTest.java | 38 ++++++ 4 files changed, 119 insertions(+), 35 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseFeaturedItemCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseFeaturedItemCommand.java index e76553dd9d4..ae2db1c752a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseFeaturedItemCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseFeaturedItemCommand.java @@ -36,7 +36,7 @@ protected void validateAndSetContent(DataverseFeaturedItem featuredItem, String this ); } - content = MarkupChecker.sanitizeBasicHTML(content); + content = MarkupChecker.sanitizeAdvancedHTML(content); if (content.length() > DataverseFeaturedItem.MAX_FEATURED_ITEM_CONTENT_SIZE) { throw new InvalidCommandArgumentsException( MessageFormat.format( diff --git a/src/main/java/edu/harvard/iq/dataverse/util/MarkupChecker.java b/src/main/java/edu/harvard/iq/dataverse/util/MarkupChecker.java index ef74819f073..02055ad60e9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/MarkupChecker.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/MarkupChecker.java @@ -1,8 +1,3 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ package edu.harvard.iq.dataverse.util; import org.apache.commons.text.StringEscapeUtils; @@ -11,56 +6,105 @@ import org.jsoup.parser.Parser; /** - * Wrapper for Jsoup clean - * + * Provides utility methods for sanitizing and processing HTML content. + *

+ * This class serves as a wrapper for the {@code Jsoup.clean} method and offers + * multiple configurations for cleaning HTML input. It also provides a method + * for escaping HTML entities and stripping all HTML tags. + *

+ * * @author rmp553 */ public class MarkupChecker { - - - + /** - * Wrapper around Jsoup clean method with the basic Safe list - * http://jsoup.org/cookbook/cleaning-html/safelist-sanitizer - * @param unsafe - * @return + * Sanitizes the provided HTML content using a customizable configuration. + *

+ * This method uses the {@code Jsoup.clean} method with a configurable {@code Safelist}. + * For more details, see the + * Jsoup SafeList Sanitizer. + *

+ *

+ * It supports preserving class attributes and optionally adding "noopener noreferrer nofollow" + * attributes to anchor tags to enhance security and usability. + *

+ * + * @param unsafe the HTML content to be sanitized; may contain unsafe or untrusted elements. + * @param keepClasses whether to preserve class attributes in the sanitized HTML. + * @param includeNoopenerNoreferrer whether to add "noopener noreferrer nofollow" to tags. + * @return a sanitized HTML string, free from potentially harmful content. */ - public static String sanitizeBasicHTML(String unsafe) { - + private static String sanitizeHTML(String unsafe, boolean keepClasses, boolean includeNoopenerNoreferrer) { if (unsafe == null) { return null; } - // basic includes: a, b, blockquote, br, cite, code, dd, dl, dt, em, i, li, ol, p, pre, q, small, span, strike, strong, sub, sup, u, ul - //Whitelist wl = Whitelist.basic().addTags("img", "h1", "h2", "h3", "kbd", "hr", "s", "del"); - Safelist sl = Safelist.basicWithImages().addTags("h1", "h2", "h3", "kbd", "hr", "s", "del", "map", "area").addAttributes("img", "usemap") - .addAttributes("map", "name").addAttributes("area", "shape", "coords", "href", "title", "alt") + // Create a base Safelist configuration + Safelist sl = Safelist.basicWithImages() + .addTags("h1", "h2", "h3", "kbd", "hr", "s", "del", "map", "area") + .addAttributes("img", "usemap") + .addAttributes("map", "name") + .addAttributes("area", "shape", "coords", "href", "title", "alt") .addEnforcedAttribute("a", "target", "_blank"); + // Add class attributes if requested + if (keepClasses) { + sl.addAttributes(":all", "class"); + } + + // Add "noopener noreferrer nofollow" to tags if requested + if (includeNoopenerNoreferrer) { + sl.addEnforcedAttribute("a", "rel", "noopener noreferrer nofollow"); + } + return Jsoup.clean(unsafe, sl); + } + /** + * Sanitizes the provided HTML content using a basic configuration. + * + * @param unsafe the HTML content to be sanitized; may contain unsafe or untrusted elements. + * @return a sanitized HTML string, free from potentially harmful content. + */ + public static String sanitizeBasicHTML(String unsafe) { + return sanitizeHTML(unsafe, false, false); } - + /** - * Strip all HTMl tags - * - * http://jsoup.org/apidocs/org/jsoup/safety/Safelist.html#none - * - * @param unsafe - * @return + * Sanitizes the provided HTML content using an advanced configuration. + *

+ * This configuration preserves class attributes and adds "noopener noreferrer nofollow" + * attributes to tags to enhance security and usability. + *

+ * + * @param unsafe the HTML content to be sanitized; may contain unsafe or untrusted elements. + * @return a sanitized HTML string, free from potentially harmful content. */ - public static String stripAllTags(String unsafe) { + public static String sanitizeAdvancedHTML(String unsafe) { + return sanitizeHTML(unsafe, true, true); + } + /** + * Removes all HTML tags from the provided content, leaving only plain text. + * + * @param unsafe the HTML content to process; may contain HTML tags. + * @return the plain text content with all HTML tags removed, or {@code null} if the input is {@code null}. + */ + public static String stripAllTags(String unsafe) { if (unsafe == null) { return null; } return Parser.unescapeEntities(Jsoup.clean(unsafe, Safelist.none()), true); - } - + + /** + * Escapes special characters in the provided string into their corresponding HTML entities. + * + * @param unsafe the string to escape; may contain special characters. + * @return a string with HTML entities escaped. + */ public static String escapeHtml(String unsafe) { - return StringEscapeUtils.escapeHtml4(unsafe); + return StringEscapeUtils.escapeHtml4(unsafe); } - } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java index d815fc77919..2fa45200977 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java @@ -78,8 +78,10 @@ public void testUpdateFeaturedItem() { verifyUpdatedFeaturedItem(updateFeatureItemResponse, "updatedTitle1", "coffeeshop.png", 2); // Update featured item: set malicious content which should be sanitized - updateFeatureItemResponse = UtilIT.updateDataverseFeaturedItem(featuredItemId, "

hello

", 2, false, "src/test/resources/images/coffeeshop.png", apiToken); - verifyUpdatedFeaturedItem(updateFeatureItemResponse, "

hello

", "coffeeshop.png", 2); + String unsafeContent = "

A title

link"; + String sanitizedContent = "

A title

link"; + updateFeatureItemResponse = UtilIT.updateDataverseFeaturedItem(featuredItemId, unsafeContent, 2, false, "src/test/resources/images/coffeeshop.png", apiToken); + verifyUpdatedFeaturedItem(updateFeatureItemResponse, sanitizedContent, "coffeeshop.png", 2); } private String createUserAndGetApiToken() { diff --git a/src/test/java/edu/harvard/iq/dataverse/util/MarkupCheckerTest.java b/src/test/java/edu/harvard/iq/dataverse/util/MarkupCheckerTest.java index 19d656b2736..02219059db3 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/MarkupCheckerTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/MarkupCheckerTest.java @@ -31,6 +31,43 @@ public void testSanitizeBasicHTML(String unsafe, String safe) { assertEquals(safe, MarkupChecker.sanitizeBasicHTML(unsafe)); } + /** + * Test of sanitizeAdvancedHTML method, of class MarkupChecker. + */ + @ParameterizedTest + @CsvSource(value = { + ", ''", + "'', ''", + // make sure we do not destroy the tags + "'\"Galactic', '\"Galactic'", + // make sure we do not destroy the tags + "'\"Galactic', '\"Galactic'", + "'

hellohello</

'", + "'

hello

', '

hello

'", + // make sure we keep text as it is when it is not html + "'Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.', 'Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.'", + // Should add noopener noreferrer attributes to tags and keep classes + "'

A title

Lorem ipsumtestlink

', '

A title

Lorem ipsumtestlink

'", + "NULL, NULL" + }, nullValues = {"NULL"}) + public void testSanitizeAdvancedHTML(String unsafe, String safe) { + String sanitizedOutput = MarkupChecker.sanitizeAdvancedHTML(unsafe); + + // Normalize both the expected and actual content by removing whitespaces + + String normalizedSafe = null; + if (safe != null) { + normalizedSafe = safe.replaceAll("\\s+", "").trim(); + } + + String normalizedOutput = null; + if (sanitizedOutput != null) { + normalizedOutput = sanitizedOutput.replaceAll("\\s+", "").trim(); + } + + assertEquals(normalizedSafe, normalizedOutput); + } + /** * Test of stripAllTags method, of class MarkupChecker. */ @@ -39,6 +76,7 @@ public void testSanitizeBasicHTML(String unsafe, String safe) { "'', ''", "NULL, NULL", "Johnson & Johnson <>, Johnson & Johnson <>", + "

Johnson & Johnson

, Johnson & Johnson", "Johnson && Johnson <&>&, Johnson && Johnson <&>&" }, nullValues = {"NULL"}) public void testStripAllTags(String unsafe, String safe) { From b83f79ca10f374eb096ff72b4603391a9c56d9b9 Mon Sep 17 00:00:00 2001 From: GPortas Date: Tue, 14 Jan 2025 12:13:10 +0000 Subject: [PATCH 091/137] Added: docs for listing and deleting all featured items --- doc/sphinx-guides/source/api/native-api.rst | 37 +++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index a363466ce57..d3e1f824c83 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1145,6 +1145,43 @@ Use the ``/settings`` API to enable or disable the enforcement of storage quotas curl -X PUT -d 'true' http://localhost:8080/api/admin/settings/:UseStorageQuotas +List Collection Featured Items +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +List the featured items configured for a given Dataverse collection ``id``: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=root + + curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/dataverses/$ID/featuredItems" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/dataverses/root/featuredItems" + +Delete All Collection Featured Items +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Deletes the featured items configured for a given Dataverse collection ``id``: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=root + + curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE "$SERVER_URL/api/dataverses/$ID/featuredItems" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/dataverses/root/featuredItems" Datasets -------- From f826063fccbcf070b39b84c2906b7a1009e994ec Mon Sep 17 00:00:00 2001 From: Dimitri Szabo <46443753+DS-INRAE@users.noreply.github.com> Date: Tue, 14 Jan 2025 18:50:17 +0100 Subject: [PATCH 092/137] Comments from review on README.md --- README.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 850a6c5eed1..55a81d904ed 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Dataverse® ## โ“ What is Dataverse? -Welcome to Dataverseยฎ, the [open source][] software platform designed for sharing, finding, citing, and preserving research data. Developed by the Dataverse team at the [Institute for Quantitative Social Science](https://iq.harvard.edu/) and the [Dataverse community][], our platform is built to support researchers and institutions in sharing, finding, citing, and preserving research their data effectively. +Welcome to Dataverseยฎ, the [open source][] software platform designed for sharing, finding, citing, and preserving research data. Developed by the Dataverse team at the [Institute for Quantitative Social Science](https://iq.harvard.edu/) and the [Dataverse community][], our platform makes it easy for research organizations to host, manage, and share their data with the world. @@ -27,7 +27,7 @@ We invite you to explore our demo site at [demo.dataverse.org][]. This site is i -## ๐ŸŒ Our Web Presence +## ๐ŸŒ Features, Integrations, Roadmaps, and More Visit [dataverse.org][], our home on the web, for a comprehensive overview of Dataverse. Here, you will find: @@ -53,7 +53,7 @@ Engage with the vibrant Dataverse community through various channels: - **[Mailing List][]**: Join the conversation on our [mailing list][]. - **[Community Calls][]**: Participate in our regular [community calls][] to discuss new features, ask questions, and share your experiences. -- **[Chat][]**: Connect with us and other users in real-time at [chat.dataverse.org][]. +- **[Chat][]**: Connect with us and other users in real-time at [dataverse.zulipchat.com][]. - **[Dataverse Community Meeting][]**: Attend our annual [Dataverse Community Meeting][] to network, learn, and collaborate with peers and experts. - **[DataverseTV][]**: Watch the video content from the Dataverse community on [DataverseTV][] and on [Harvard's IQSS YouTube channel][]. @@ -74,7 +74,7 @@ Dataverse is a trademark of President and Fellows of Harvard College and is regi --- For more detailed information, visit our website at [dataverse.org][]. -Feel free to reach out with any questions or feedback. Happy researching! +Feel free to [reach out] with any questions or feedback. Happy researching! [![Dataverse Project logo](src/main/webapp/resources/images/dataverseproject_logo.jpg "Dataverse Project")](http://dataverse.org) @@ -98,10 +98,11 @@ Feel free to reach out with any questions or feedback. Happy researching! [Contributing Guide]: CONTRIBUTING.md [mailing list]: https://groups.google.com/group/dataverse-community [community call]: https://dataverse.org/community-calls -[Chat]: https://chat.dataverse.org -[chat.dataverse.org]: https://chat.dataverse.org +[Chat]: https://dataverse.zulipchat.com +[dataverse.zulipchat.com]: https://dataverse.zulipchat.com [Dataverse Community Meeting]: https://dataverse.org/events [open source]: LICENSE.md [community calls]: https://dataverse.org/community-calls [DataverseTV]: https://dataverse.org/dataversetv [Harvard's IQSS YouTube channel]: https://www.youtube.com/@iqssatharvarduniversity8672 +[reach out]: https://dataverse.org/contact From e0067807a363d21f24b1dcd9eb74d8c7017ed45e Mon Sep 17 00:00:00 2001 From: GPortas Date: Wed, 15 Jan 2025 13:25:01 +0000 Subject: [PATCH 093/137] Added: API docs for creating a collection featured item --- doc/sphinx-guides/source/api/native-api.rst | 24 +++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index d3e1f824c83..c4d75377769 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1183,6 +1183,30 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/dataverses/root/featuredItems" +Create A Collection Featured Item +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Creates a featured item in the given Dataverse collection ``id``: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export IMAGE_FILENAME='image.png' + export CONTENT='Content for featured item.' + export DISPLAY_ORDER=1 + export SERVER_URL=https://demo.dataverse.org + export ID=root + + curl -H "X-Dataverse-key:$API_TOKEN" -X POST -F "file=@$IMAGE_FILENAME" -F "content=$CONTENT" -F "displayOrder=$DISPLAY_ORDER" "$SERVER_URL/api/dataverses/$ID/featuredItems" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:0da2519d-eea5-49ac-bb5d-c349e2e36503" -X POST -F "file=@image.png" -F "content=Content for featured item." -F "displayOrder=1" "https://demo.dataverse.org/api/dataverses/root/featuredItems" + +A featured item may or may not contain an image. If you wish to create it without an image, simply omit the file parameter in the request. + Datasets -------- From a5b268706a7fc66576e9774f7a9cf3c469dc6891 Mon Sep 17 00:00:00 2001 From: GPortas Date: Wed, 15 Jan 2025 16:32:27 +0000 Subject: [PATCH 094/137] Added: docs for updating a single featured item through API --- doc/sphinx-guides/source/api/native-api.rst | 43 ++++++++++++++++++++- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 12d51166cf4..a2c36c4a995 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1214,9 +1214,48 @@ The fully expanded example above (without environment variables) looks like this .. code-block:: bash - curl -H "X-Dataverse-key:0da2519d-eea5-49ac-bb5d-c349e2e36503" -X POST -F "file=@image.png" -F "content=Content for featured item." -F "displayOrder=1" "https://demo.dataverse.org/api/dataverses/root/featuredItems" + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST -F "file=@image.png" -F "content=Content for featured item." -F "displayOrder=1" "https://demo.dataverse.org/api/dataverses/root/featuredItems" -A featured item may or may not contain an image. If you wish to create it without an image, simply omit the file parameter in the request. +A featured item may or may not contain an image. If you wish to create it without an image, omit the file parameter in the request. + +Dataverse Collection Featured Items +----------------------------------- + +Update A Collection Featured Item +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Updates a featured item given its ``id``: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export IMAGE_FILENAME='image.png' + export CONTENT='Content for featured item.' + export DISPLAY_ORDER=1 + export SERVER_URL=https://demo.dataverse.org + export ID=1 + + curl -H "X-Dataverse-key:$API_TOKEN" -X PUT -F "file=@$IMAGE_FILENAME" -F "content=$CONTENT" -F "displayOrder=$DISPLAY_ORDER" "$SERVER_URL/api/api/dataverseFeaturedItems/2" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT -F "file=@image.png" -F "content=Content for featured item." -F "displayOrder=1" "https://demo.dataverse.org/api/dataverseFeaturedItems/1" + +``content`` and ``displayOrder`` must always be provided; otherwise, an error will occur. Use the ``file`` parameter to set a new image for the featured item. To keep the existing image, omit ``file`` and send ``keepFile=true``. To remove the image, omit the file parameter. + +Updating the featured item keeping the existing image: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT -F "keepFile=true" -F "content=Content for featured item." -F "displayOrder=1" "https://demo.dataverse.org/api/dataverseFeaturedItems/1" + +Updating the featured item removing the existing image: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT -F "content=Content for featured item." -F "displayOrder=1" "https://demo.dataverse.org/api/dataverseFeaturedItems/1" Datasets -------- From 651347e050d836fd2fc59c4b2e998eadcc1a27fd Mon Sep 17 00:00:00 2001 From: GPortas Date: Wed, 15 Jan 2025 16:39:41 +0000 Subject: [PATCH 095/137] Added: docs for deleting a single featured item through API --- doc/sphinx-guides/source/api/native-api.rst | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index a2c36c4a995..98321f15de7 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1235,7 +1235,7 @@ Updates a featured item given its ``id``: export SERVER_URL=https://demo.dataverse.org export ID=1 - curl -H "X-Dataverse-key:$API_TOKEN" -X PUT -F "file=@$IMAGE_FILENAME" -F "content=$CONTENT" -F "displayOrder=$DISPLAY_ORDER" "$SERVER_URL/api/api/dataverseFeaturedItems/2" + curl -H "X-Dataverse-key:$API_TOKEN" -X PUT -F "file=@$IMAGE_FILENAME" -F "content=$CONTENT" -F "displayOrder=$DISPLAY_ORDER" "$SERVER_URL/api/dataverseFeaturedItems/$ID" The fully expanded example above (without environment variables) looks like this: @@ -1257,6 +1257,25 @@ Updating the featured item removing the existing image: curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT -F "content=Content for featured item." -F "displayOrder=1" "https://demo.dataverse.org/api/dataverseFeaturedItems/1" +Delete A Collection Featured Item +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Deletes a featured item given its ``id``: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=1 + + curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE "$SERVER_URL/api/dataverseFeaturedItems/$ID" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/dataverseFeaturedItems/1" + Datasets -------- From 4503aa896b808b55d7c9e1e84bf53d2e5861ce8a Mon Sep 17 00:00:00 2001 From: GPortas Date: Wed, 15 Jan 2025 16:55:15 +0000 Subject: [PATCH 096/137] Fixed: Bundle.properties string format --- src/main/java/propertyFiles/Bundle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 441a489ed3b..085761bff56 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -3127,5 +3127,5 @@ authenticationServiceBean.errors.invalidBearerToken=Could not parse bearer token authenticationServiceBean.errors.bearerTokenDetectedNoOIDCProviderConfigured=Bearer token detected, no OIDC provider configured. #DataverseFeaturedItems.java -dataverseFeaturedItems.errors.notFound=Can't find dataverse featured item with identifier {0} +dataverseFeaturedItems.errors.notFound=Could not find dataverse featured item with identifier {0} dataverseFeaturedItems.delete.successful=Successfully deleted dataverse featured item with identifier {0} From 4c64eaaf66721acc904d3da0e18f641051a6709a Mon Sep 17 00:00:00 2001 From: Vera Clemens Date: Thu, 16 Jan 2025 10:58:42 +0100 Subject: [PATCH 097/137] test: add failing test to reproduce 500 error for editMetadata API --- .../dataset-add-single-cvoc-field-metadata.json | 4 ++++ .../edu/harvard/iq/dataverse/api/DatasetsIT.java | 16 +++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 doc/sphinx-guides/source/_static/api/dataset-add-single-cvoc-field-metadata.json diff --git a/doc/sphinx-guides/source/_static/api/dataset-add-single-cvoc-field-metadata.json b/doc/sphinx-guides/source/_static/api/dataset-add-single-cvoc-field-metadata.json new file mode 100644 index 00000000000..620f3df10d1 --- /dev/null +++ b/doc/sphinx-guides/source/_static/api/dataset-add-single-cvoc-field-metadata.json @@ -0,0 +1,4 @@ +{ + "typeName": "journalArticleType", + "value": "abstract" +} \ No newline at end of file diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index e26064a24ef..7c6766254d0 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -26,6 +26,7 @@ import jakarta.json.JsonArray; import jakarta.json.JsonObject; import jakarta.json.JsonObjectBuilder; +import jakarta.json.JsonArrayBuilder; import jakarta.ws.rs.core.Response.Status; import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; @@ -368,6 +369,13 @@ public void testAddUpdateDatasetViaNativeAPI() { createDataverseResponse.prettyPrint(); String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + JsonArrayBuilder metadataBlocks = Json.createArrayBuilder(); + metadataBlocks.add("citation"); + metadataBlocks.add("journal"); + Response setMetadataBlocksResponse = UtilIT.setMetadataBlocks(dataverseAlias, metadataBlocks, apiToken); + setMetadataBlocksResponse.prettyPrint(); + setMetadataBlocksResponse.then().assertThat().statusCode(OK.getStatusCode()); + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); createDatasetResponse.prettyPrint(); Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse); @@ -411,7 +419,13 @@ public void testAddUpdateDatasetViaNativeAPI() { String responseString = addSubjectSingleViaNative.prettyPrint(); addSubjectSingleViaNative.then().assertThat() .statusCode(OK.getStatusCode()).body(containsString("Mathematical Sciences")).body(containsString("Social Sciences")); - + + + String pathToJsonFileSingleCvoc = "doc/sphinx-guides/source/_static/api/dataset-add-single-cvoc-field-metadata.json"; + Response addSingleCvocViaNative = UtilIT.updateFieldLevelDatasetMetadataViaNative(datasetPersistentId, pathToJsonFileSingleCvoc, apiToken); + addSingleCvocViaNative.prettyPrint(); + addSingleCvocViaNative.then().assertThat() + .statusCode(OK.getStatusCode()); //Trying to blank out required field should fail... From 15568a87f83c97a831a887515b870869d10fe0ee Mon Sep 17 00:00:00 2001 From: GPortas Date: Thu, 16 Jan 2025 13:21:57 +0000 Subject: [PATCH 098/137] Fixed: typos in Bundle.properties --- src/main/java/propertyFiles/Bundle.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 085761bff56..a0b5cff2c49 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -992,8 +992,8 @@ dataverse.create.featuredItem.error.fileSizeExceedsLimit=File exceeds the maximu dataverse.create.featuredItem.error.invalidFileType=Invalid image file type dataverse.create.featuredItem.error.contentShouldBeProvided=Featured item 'content' property should be provided and not empty. dataverse.create.featuredItem.error.contentExceedsLengthLimit=Featured item content exceeds the maximum allowed length of {0} characters. -dataverse.update.featuredItems.error.missingInputParams=All input parameters (id, content, displayOrder, keepFile, fileNames) are required. -dataverse.update.featuredItems.error.inputListsSizeMismatch=All input lists (id, content, displayOrder, keepFile, fileNames) must have the same size. +dataverse.update.featuredItems.error.missingInputParams=All input parameters (id, content, displayOrder, keepFile, fileName) are required. +dataverse.update.featuredItems.error.inputListsSizeMismatch=All input lists (id, content, displayOrder, keepFile, fileName) must have the same size. dataverse.delete.featuredItems.success=All featured items of this Dataverse have been successfully deleted. # rolesAndPermissionsFragment.xhtml From 93c1eca0b59dec5511e51bee31d181c6136bb7dd Mon Sep 17 00:00:00 2001 From: GPortas Date: Thu, 16 Jan 2025 15:36:12 +0000 Subject: [PATCH 099/137] Added: API docs for updating all featured items --- doc/sphinx-guides/source/api/native-api.rst | 68 +++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index b19d7cff668..a30a17054d2 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1218,6 +1218,74 @@ The fully expanded example above (without environment variables) looks like this A featured item may or may not contain an image. If you wish to create it without an image, omit the file parameter in the request. +Update All Collection Featured Items +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Updates all featured items in the given Dataverse collection ``id``. + +The data sent to the endpoint represents the desired final state of the featured items in the Dataverse collection and overwrites any existing featured items configuration. + +The parameters ``id``, ``content``, ``displayOrder``, and ``fileName`` must be specified as many times as the number of items we want to add or update. The order in which these parameters are repeated must match to ensure they correspond to the same featured item. + +The ``file`` parameter must be specified for each image we want to attach to featured items. Note that images can be shared between featured items, so ``fileName`` can have the same value in different featured items. + +The ``id`` parameter must be ``0`` for new items or set to the item's identifier for updates. The ``fileName`` parameter should be empty to exclude an image or match the name of a file sent in a ``file`` parameter to set a new image. ``keepFile`` must always be set to ``false``, unless it's an update to a featured item where we want to preserve the existing image, if one exists. + +Note that any existing featured item not included in the call with its associated identifier and corresponding properties will be removed from the collection. + +The following example creates two featured items, with an image assigned to the second one: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=root + + export FIRST_ITEM_CONTENT='Content 1' + export FIRST_ITEM_DISPLAY_ORDER=1 + + export SECOND_ITEM_IMAGE_FILENAME='image.png' + export SECOND_ITEM_CONTENT='Content 2' + export SECOND_ITEM_DISPLAY_ORDER=2 + + curl -H "X-Dataverse-key:$API_TOKEN" \ + -X PUT \ + -F "id=0" -F "id=0" \ + -F "content=$FIRST_ITEM_CONTENT" -F "content=$SECOND_ITEM_CONTENT" \ + -F "displayOrder=$FIRST_ITEM_DISPLAY_ORDER" -F "displayOrder=$SECOND_ITEM_DISPLAY_ORDER" \ + -F "fileName=" -F "fileName=$SECOND_ITEM_IMAGE_FILENAME" \ + -F "keepFile=false" -F "keepFile=false" \ + -F "file=@$SECOND_ITEM_IMAGE_FILENAME" \ + "$SERVER_URL/api/dataverses/$ID/featuredItems" + + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" \ + -X PUT \ + -F "id=0" -F "id=0" \ + -F "content=Content 1" -F "content=Content 2" \ + -F "displayOrder=1" -F "displayOrder=2" \ + -F "fileName=" -F "fileName=image.png" \ + -F "keepFile=false" -F "keepFile=false" \ + -F "file=@image.png" \ + "https://demo.dataverse.org/api/dataverses/root/featuredItems" + +The following example creates one featured item and updates a second one, keeping the existing image it may have had: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" \ + -X PUT \ + -F "id=0" -F "id=1" \ + -F "content=Content 1" -F "content=Updated content 2" \ + -F "displayOrder=1" -F "displayOrder=2" \ + -F "fileName=" -F "fileName=" \ + -F "keepFile=false" -F "keepFile=true" \ + "https://demo.dataverse.org/api/dataverses/root/featuredItems" + Dataverse Collection Featured Items ----------------------------------- From 13301be341da84b9512c636d7de4f24d4ea959ad Mon Sep 17 00:00:00 2001 From: GPortas Date: Thu, 16 Jan 2025 15:46:19 +0000 Subject: [PATCH 100/137] Added: release notes for #10943 --- doc/release-notes/10943-featured-items.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/release-notes/10943-featured-items.md diff --git a/doc/release-notes/10943-featured-items.md b/doc/release-notes/10943-featured-items.md new file mode 100644 index 00000000000..0b485fdb30a --- /dev/null +++ b/doc/release-notes/10943-featured-items.md @@ -0,0 +1,8 @@ +CRUD endpoints for Collection Featured Items have been implemented. In particular, the following endpoints have been implemented: + +- Create a feature item (POST /api/dataverses//featuredItems) +- Update a feature item (PUT /api/dataverseFeaturedItems/) +- Delete a feature item (DELETE /api/dataverseFeaturedItems/) +- List all featured items in a collection (GET /api/dataverses//featuredItems) +- Delete all featured items in a collection (DELETE /api/dataverses//featuredItems) +- Update all featured items in a collection (PUT /api/dataverses//featuredItems) From 38e7d88f81a1b08022cee40b91c68ee06e39d56d Mon Sep 17 00:00:00 2001 From: GPortas Date: Thu, 16 Jan 2025 15:53:32 +0000 Subject: [PATCH 101/137] Changed: API endpoint route --- src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java | 2 +- src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index f5abaa82004..8cfe9b9bcf6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -1778,7 +1778,7 @@ public Response getUserPermissionsOnDataverse(@Context ContainerRequestContext c @POST @AuthRequired @Consumes(MediaType.MULTIPART_FORM_DATA) - @Path("{identifier}/featuredItem") + @Path("{identifier}/featuredItems") public Response createFeaturedItem(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf, @FormDataParam("content") String content, diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 3f3e08f1a6b..cd089db3863 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -4441,7 +4441,7 @@ static Response createDataverseFeaturedItem(String dataverseAlias, return requestSpecification .when() - .post("/api/dataverses/" + dataverseAlias + "/featuredItem"); + .post("/api/dataverses/" + dataverseAlias + "/featuredItems"); } static Response deleteDataverseFeaturedItem(long id, String apiToken) { From f5717dc8eae1f7f3f1a22f55acf6590a903d58f1 Mon Sep 17 00:00:00 2001 From: Vera Clemens Date: Fri, 17 Jan 2025 16:42:24 +0100 Subject: [PATCH 102/137] fix: fix 500 error when uploading single cvoc values to editMetadata API --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 0fe9099a3e4..c2e777d76f5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -1187,8 +1187,12 @@ private String validateDatasetFieldValues(List fields) { if (dsf.getDatasetFieldType().isAllowMultiples() && dsf.getControlledVocabularyValues().isEmpty() && dsf.getDatasetFieldCompoundValues().isEmpty() && dsf.getDatasetFieldValues().isEmpty()) { error.append("Empty multiple value for field: ").append(dsf.getDatasetFieldType().getDisplayName()).append(" "); - } else if (!dsf.getDatasetFieldType().isAllowMultiples() && dsf.getSingleValue().getValue().isEmpty()) { - error.append("Empty value for field: ").append(dsf.getDatasetFieldType().getDisplayName()).append(" "); + } else if (!dsf.getDatasetFieldType().isAllowMultiples()) { + if (!dsf.getDatasetFieldType().isControlledVocabulary() && dsf.getSingleValue().getValue().isEmpty()) { + error.append("Empty value for field: ").append(dsf.getDatasetFieldType().getDisplayName()).append(" "); + } else if (dsf.getDatasetFieldType().isControlledVocabulary() && dsf.getSingleControlledVocabularyValue().getStrValue().isEmpty()) { + error.append("Empty cvoc value for field: ").append(dsf.getDatasetFieldType().getDisplayName()).append(" "); + } } } From ac1a881ff9d6a69f748235d3445e4b5a8bcfb52d Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 17 Jan 2025 15:15:19 -0500 Subject: [PATCH 103/137] add test asserting current ROR regex behavior #11075 --- .../iq/dataverse/DatasetFieldValueValidatorTest.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/test/java/edu/harvard/iq/dataverse/DatasetFieldValueValidatorTest.java b/src/test/java/edu/harvard/iq/dataverse/DatasetFieldValueValidatorTest.java index b753f534c6b..da9496e899e 100644 --- a/src/test/java/edu/harvard/iq/dataverse/DatasetFieldValueValidatorTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/DatasetFieldValueValidatorTest.java @@ -157,6 +157,14 @@ public void testIsValidAuthorIdentifierGnd() { assertFalse(validator.isValidAuthorIdentifier("junk", pattern)); } + @Test + public void testIsValidAuthorIdentifierRor() { + DatasetFieldValueValidator validator = new DatasetFieldValueValidator(); + Pattern pattern = ExternalIdentifier.valueOf("ROR").getPattern(); + assertTrue(validator.isValidAuthorIdentifier("https://ror.org/03vek6s52", pattern)); + assertFalse(validator.isValidAuthorIdentifier("junk", pattern)); + } + final Validator validator = Validation.buildDefaultValidatorFactory().getValidator(); @ParameterizedTest From 31949f8f78f95ba2430d65e84023963812d131b6 Mon Sep 17 00:00:00 2001 From: Dimitri Szabo <46443753+DS-INRAE@users.noreply.github.com> Date: Mon, 20 Jan 2025 09:09:41 +0100 Subject: [PATCH 104/137] Update README ToC --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 55a81d904ed..11dc1ae4fa7 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Dataverse® 1. [โ“ What is Dataverse?](#what-is-dataverse) 2. [โœ” Try Dataverse](#try-dataverse) -3. [๐ŸŒ Our Web Presence](#our-web-presence) +3. [๐ŸŒ Features, Integrations, Roadmaps, and More](#our-web-presence) 4. [๐Ÿ“ฅ Installation](#installation) 5. [๐Ÿ˜ Community and Support](#community-and-support) 6. [๐Ÿง‘โ€๐Ÿ’ป๏ธ Contributing](#contributing) From 2834dbf1bd10d9f2de75d36c66a62c10fb8f1993 Mon Sep 17 00:00:00 2001 From: Dimitri Szabo <46443753+DS-INRAE@users.noreply.github.com> Date: Mon, 20 Jan 2025 09:10:55 +0100 Subject: [PATCH 105/137] Update web presence anchor --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 11dc1ae4fa7..2303c001d2c 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Dataverse® 1. [โ“ What is Dataverse?](#what-is-dataverse) 2. [โœ” Try Dataverse](#try-dataverse) -3. [๐ŸŒ Features, Integrations, Roadmaps, and More](#our-web-presence) +3. [๐ŸŒ Features, Integrations, Roadmaps, and More](#website) 4. [๐Ÿ“ฅ Installation](#installation) 5. [๐Ÿ˜ Community and Support](#community-and-support) 6. [๐Ÿง‘โ€๐Ÿ’ป๏ธ Contributing](#contributing) @@ -25,7 +25,7 @@ Welcome to Dataverseยฎ, the [open source][] software platform designed for shari We invite you to explore our demo site at [demo.dataverse.org][]. This site is ideal for testing and evaluating Dataverse in a risk-free environment. - + ## ๐ŸŒ Features, Integrations, Roadmaps, and More From 866f245115c04d4d2dfefab06cc20a4e29dc61fb Mon Sep 17 00:00:00 2001 From: Vera Clemens Date: Mon, 20 Jan 2025 10:36:19 +0100 Subject: [PATCH 106/137] test: add failing test to reproduce 500 error for editMetadata API --- .../dataset-add-single-compound-field-metadata.json | 13 +++++++++++++ .../edu/harvard/iq/dataverse/api/DatasetsIT.java | 8 ++++++++ 2 files changed, 21 insertions(+) create mode 100644 doc/sphinx-guides/source/_static/api/dataset-add-single-compound-field-metadata.json diff --git a/doc/sphinx-guides/source/_static/api/dataset-add-single-compound-field-metadata.json b/doc/sphinx-guides/source/_static/api/dataset-add-single-compound-field-metadata.json new file mode 100644 index 00000000000..f49a9e47d5b --- /dev/null +++ b/doc/sphinx-guides/source/_static/api/dataset-add-single-compound-field-metadata.json @@ -0,0 +1,13 @@ +{ + "fields": [ + { + "typeName": "targetSampleSize", + "value": { + "targetSampleActualSize": { + "typeName": "targetSampleSizeFormula", + "value": "n = N*X / (X + N โ€“ 1)" + } + } + } + ] +} \ No newline at end of file diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 7c6766254d0..72cc68f00bb 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -372,6 +372,7 @@ public void testAddUpdateDatasetViaNativeAPI() { JsonArrayBuilder metadataBlocks = Json.createArrayBuilder(); metadataBlocks.add("citation"); metadataBlocks.add("journal"); + metadataBlocks.add("socialscience"); Response setMetadataBlocksResponse = UtilIT.setMetadataBlocks(dataverseAlias, metadataBlocks, apiToken); setMetadataBlocksResponse.prettyPrint(); setMetadataBlocksResponse.then().assertThat().statusCode(OK.getStatusCode()); @@ -428,6 +429,13 @@ public void testAddUpdateDatasetViaNativeAPI() { .statusCode(OK.getStatusCode()); + String pathToJsonFileSingleCompound = "doc/sphinx-guides/source/_static/api/dataset-add-single-compound-field-metadata.json"; + Response addSingleCompoundViaNative = UtilIT.updateFieldLevelDatasetMetadataViaNative(datasetPersistentId, pathToJsonFileSingleCompound, apiToken); + addSingleCompoundViaNative.prettyPrint(); + addSingleCompoundViaNative.then().assertThat() + .statusCode(OK.getStatusCode()); + + //Trying to blank out required field should fail... String pathToJsonFileBadData = "doc/sphinx-guides/source/_static/api/dataset-update-with-blank-metadata.json"; Response deleteTitleViaNative = UtilIT.updateFieldLevelDatasetMetadataViaNative(datasetPersistentId, pathToJsonFileBadData, apiToken); From 69b9a0163202dc4ede53ea5513ca73761835cffa Mon Sep 17 00:00:00 2001 From: Vera Clemens Date: Mon, 20 Jan 2025 10:52:42 +0100 Subject: [PATCH 107/137] fix: fix 500 error when uploading single compound values to editMetadata API --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index c2e777d76f5..5ccec629ee4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -1188,10 +1188,12 @@ private String validateDatasetFieldValues(List fields) { && dsf.getDatasetFieldCompoundValues().isEmpty() && dsf.getDatasetFieldValues().isEmpty()) { error.append("Empty multiple value for field: ").append(dsf.getDatasetFieldType().getDisplayName()).append(" "); } else if (!dsf.getDatasetFieldType().isAllowMultiples()) { - if (!dsf.getDatasetFieldType().isControlledVocabulary() && dsf.getSingleValue().getValue().isEmpty()) { - error.append("Empty value for field: ").append(dsf.getDatasetFieldType().getDisplayName()).append(" "); - } else if (dsf.getDatasetFieldType().isControlledVocabulary() && dsf.getSingleControlledVocabularyValue().getStrValue().isEmpty()) { + if (dsf.getDatasetFieldType().isControlledVocabulary() && dsf.getSingleControlledVocabularyValue().getStrValue().isEmpty()) { error.append("Empty cvoc value for field: ").append(dsf.getDatasetFieldType().getDisplayName()).append(" "); + } else if (dsf.getDatasetFieldType().isCompound() && dsf.getDatasetFieldCompoundValues().isEmpty()) { + error.append("Empty compound value for field: ").append(dsf.getDatasetFieldType().getDisplayName()).append(" "); + } else if (!dsf.getDatasetFieldType().isControlledVocabulary() && !dsf.getDatasetFieldType().isCompound() && dsf.getSingleValue().getValue().isEmpty()) { + error.append("Empty value for field: ").append(dsf.getDatasetFieldType().getDisplayName()).append(" "); } } } From 36d45896d073035dfe796e12e190df50cb1f9897 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 21 Jan 2025 09:23:50 -0500 Subject: [PATCH 108/137] add DataCite export assertions for authors with RORs #11075 --- .../doi/datacite/XmlMetadataTemplateTest.java | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/test/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/XmlMetadataTemplateTest.java b/src/test/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/XmlMetadataTemplateTest.java index 2bd6818821d..7faa9a0a6fa 100644 --- a/src/test/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/XmlMetadataTemplateTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/XmlMetadataTemplateTest.java @@ -34,6 +34,7 @@ import edu.harvard.iq.dataverse.util.testing.JvmSetting; import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings; import edu.harvard.iq.dataverse.util.xml.XmlValidator; +import io.restassured.path.xml.XmlPath; import jakarta.json.JsonArray; import jakarta.json.JsonObject; import jakarta.json.JsonString; @@ -113,6 +114,8 @@ public void testDataCiteXMLCreation() throws IOException { df2.setDatasetFieldType(dft2); df2.setSingleValue("Harvard University"); alice.setAffiliation(df2); + alice.setIdType("ORCID"); + alice.setIdValue("0000-0002-1825-0097"); DatasetAuthor bob = new DatasetAuthor(); DatasetField df3 = new DatasetField(); df3.setDatasetFieldType(dft); @@ -122,9 +125,27 @@ public void testDataCiteXMLCreation() throws IOException { df4.setDatasetFieldType(dft2); df4.setSingleValue("QDR"); bob.setAffiliation(df4); + DatasetAuthor harvard = new DatasetAuthor(); + DatasetField df5 = new DatasetField(); + df5.setDatasetFieldType(dft); + df5.setSingleValue("Harvard University"); + harvard.setName(df5); + harvard.setIdType("ROR"); + harvard.setIdValue("03vek6s52"); + DatasetAuthor qdr = new DatasetAuthor(); + DatasetField df6 = new DatasetField(); + df6.setDatasetFieldType(dft); + df6.setSingleValue("Qualitative Data Repository"); + qdr.setName(df6); + qdr.setIdType("ROR"); + // This value is set improperly as a URL. It should be just + // the identifier (014trz974) as in the ORCID example above. + qdr.setIdValue("https://ror.org/014trz974"); List authors = new ArrayList<>(); authors.add(alice); authors.add(bob); + authors.add(harvard); + authors.add(qdr); doiMetadata.setAuthors(authors); doiMetadata.setPublisher("Dataverse"); XmlMetadataTemplate template = new XmlMetadataTemplate(doiMetadata); @@ -167,6 +188,23 @@ public void testDataCiteXMLCreation() throws IOException { System.out.println("Invalid schema: " + e.getMessage()); } + assertEquals("Alice", XmlPath.from(xml).getString("resource.creators.creator[0].creatorName")); + assertEquals("https://orcid.org/0000-0002-1825-0097", XmlPath.from(xml).getString("resource.creators.creator[0].nameIdentifier")); + assertEquals("ORCID", XmlPath.from(xml).getString("resource.creators.creator[0].nameIdentifier.@nameIdentifierScheme")); + assertEquals("https://orcid.org", XmlPath.from(xml).getString("resource.creators.creator[0].nameIdentifier.@schemeURI")); + assertEquals("Bob", XmlPath.from(xml).getString("resource.creators.creator[1].creatorName")); + assertEquals("Harvard University", XmlPath.from(xml).getString("resource.creators.creator[2].creatorName")); + // FIXME: nameIdentifierScheme, nameIdentifierScheme, and schemeURI should be populated + assertEquals("", XmlPath.from(xml).getString("resource.creators.creator[2].nameIdentifier")); + assertEquals(null, XmlPath.from(xml).getString("resource.creators.creator[2].nameIdentifier.@nameIdentifierScheme")); + assertEquals(null, XmlPath.from(xml).getString("resource.creators.creator[2].nameIdentifier.@schemeURI")); + assertEquals("Qualitative Data Repository", XmlPath.from(xml).getString("resource.creators.creator[3].creatorName")); + // FIXME: the URL below is not right + assertEquals("https://ror.org/https://ror.org/014trz974", XmlPath.from(xml).getString("resource.creators.creator[3].nameIdentifier")); + assertEquals("ROR", XmlPath.from(xml).getString("resource.creators.creator[3].nameIdentifier.@nameIdentifierScheme")); + assertEquals("https://ror.org", XmlPath.from(xml).getString("resource.creators.creator[3].nameIdentifier.@schemeURI")); + assertEquals("Dataverse", XmlPath.from(xml).getString("resource.publisher")); + } /** From 782188fcf4485fd65f15b287aba8d2dfde904778 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 21 Jan 2025 09:59:06 -0500 Subject: [PATCH 109/137] ROR url not added in affiliation --- .../iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 8199b7d9c9f..e8412728fbf 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -639,7 +639,7 @@ private void writeEntityElements(XMLStreamWriter xmlw, String elementName, Strin attributeMap.put("schemeURI", "https://ror.org"); attributeMap.put("affiliationIdentifierScheme", "ROR"); - attributeMap.put("affiliationIdentifier", orgName); + attributeMap.put("affiliationIdentifier", affiliation); } XmlWriterUtil.writeFullElementWithAttributes(xmlw, "affiliation", attributeMap, StringEscapeUtils.escapeXml10(orgName)); From c4387d59f095de7891a8491da9c9136b3fd6f061 Mon Sep 17 00:00:00 2001 From: Guillermo Portas Date: Tue, 21 Jan 2025 18:59:28 +0000 Subject: [PATCH 110/137] Changed: updated release notes Co-authored-by: Philip Durbin --- doc/release-notes/10943-featured-items.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/release-notes/10943-featured-items.md b/doc/release-notes/10943-featured-items.md index 0b485fdb30a..e7778a2f9c9 100644 --- a/doc/release-notes/10943-featured-items.md +++ b/doc/release-notes/10943-featured-items.md @@ -6,3 +6,5 @@ CRUD endpoints for Collection Featured Items have been implemented. In particula - List all featured items in a collection (GET /api/dataverses//featuredItems) - Delete all featured items in a collection (DELETE /api/dataverses//featuredItems) - Update all featured items in a collection (PUT /api/dataverses//featuredItems) + +See also #10943 and #11124. From 17f523af6209188ed92066022be839626d09d277 Mon Sep 17 00:00:00 2001 From: Guillermo Portas Date: Tue, 21 Jan 2025 19:00:14 +0000 Subject: [PATCH 111/137] Changed: docs title format tweak Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index a30a17054d2..e779dac3779 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1194,7 +1194,7 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/dataverses/root/featuredItems" -Create A Collection Featured Item +Create a Collection Featured Item ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Creates a featured item in the given Dataverse collection ``id``: From 746d5834399e73beaa07847e85e33e50765a57dc Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 21 Jan 2025 14:55:19 -0500 Subject: [PATCH 112/137] add ROR tests for author affiliation and funder #11075 --- .../harvard/iq/dataverse/api/DatasetsIT.java | 172 ++++++++++++++++++ 1 file changed, 172 insertions(+) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index e26064a24ef..7f2569982f3 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -4303,6 +4303,178 @@ public void testCitationDate() throws IOException { .statusCode(OK.getStatusCode()); } + @Test + public void testDataCiteExport() throws IOException { + + Response createUser = UtilIT.createRandomUser(); + createUser.then().assertThat().statusCode(OK.getStatusCode()); + String username = UtilIT.getUsernameFromResponse(createUser); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response createDataverse = UtilIT.createRandomDataverse(apiToken); + createDataverse.then().assertThat().statusCode(CREATED.getStatusCode()); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverse); + Integer dataverseId = UtilIT.getDataverseIdFromResponse(createDataverse); + + JsonObjectBuilder datasetJson = Json.createObjectBuilder() + .add("datasetVersion", Json.createObjectBuilder() + .add("license", Json.createObjectBuilder() + .add("name", "CC0 1.0") + .add("uri", "http://creativecommons.org/publicdomain/zero/1.0") + ) + .add("metadataBlocks", Json.createObjectBuilder() + .add("citation", Json.createObjectBuilder() + .add("fields", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("typeName", "title") + .add("value", "Test dataset") + .add("typeClass", "primitive") + .add("multiple", false) + ) + .add(Json.createObjectBuilder() + .add("value", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("authorName", + Json.createObjectBuilder() + .add("value", "Simpson, Homer") + .add("typeClass", "primitive") + .add("multiple", false) + .add("typeName", "authorName") + ) + .add("authorAffiliation", + Json.createObjectBuilder() + .add("value", "https://ror.org/03vek6s52") + .add("typeClass", "primitive") + .add("multiple", false) + .add("typeName", "authorAffiliation") + ) + ) + ) + .add("typeClass", "compound") + .add("multiple", true) + .add("typeName", "author") + ) + .add(Json.createObjectBuilder() + .add("value", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("datasetContactEmail", + Json.createObjectBuilder() + .add("value", "hsimpson@mailinator.com") + .add("typeClass", "primitive") + .add("multiple", false) + .add("typeName", "datasetContactEmail")) + ) + ) + .add("typeClass", "compound") + .add("multiple", true) + .add("typeName", "datasetContact") + ) + .add(Json.createObjectBuilder() + .add("value", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("dsDescriptionValue", + Json.createObjectBuilder() + .add("value", "Just a test dataset.") + .add("typeClass", "primitive") + .add("multiple", false) + .add("typeName", "dsDescriptionValue")) + ) + ) + .add("typeClass", "compound") + .add("multiple", true) + .add("typeName", "dsDescription") + ) + .add(Json.createObjectBuilder() + .add("value", Json.createArrayBuilder() + .add("Other") + ) + .add("typeClass", "controlledVocabulary") + .add("multiple", true) + .add("typeName", "subject") + ) + .add(Json.createObjectBuilder() + .add("value", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("authorName", + Json.createObjectBuilder() + .add("value", "https://ror.org/05h1kgg64") // NIH + .add("typeClass", "primitive") + .add("multiple", false) + .add("typeName", "grantNumberAgency") + ) + .add("authorAffiliation", + Json.createObjectBuilder() + .add("value", "12345") + .add("typeClass", "primitive") + .add("multiple", false) + .add("typeName", "grantNumberValue") + ) + ) + ) + .add("typeClass", "compound") + .add("multiple", true) + .add("typeName", "grantNumber") + ) + ) + ) + )); + + Response createDatasetResponse = UtilIT.createDataset(dataverseAlias, datasetJson, apiToken); + createDatasetResponse.prettyPrint(); + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDatasetResponse); + String datasetPid = JsonPath.from(createDatasetResponse.getBody().asString()).getString("data.persistentId"); + + Response publishDataverse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken); + publishDataverse.prettyPrint(); + publishDataverse.then().assertThat().statusCode(OK.getStatusCode()); + Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken); + publishDataset.prettyPrint(); + publishDataset.then().assertThat().statusCode(OK.getStatusCode()); + + Response exportDatasetAsDataCite = UtilIT.exportDataset(datasetPid, "Datacite", apiToken, true); + exportDatasetAsDataCite.prettyPrint(); + exportDatasetAsDataCite.then().assertThat() + .body("resource.creators.creator[0].creatorName", equalTo("Simpson, Homer")) + // see below for additional affiliation assertions, which can vary + .body("resource.creators.creator[0].affiliation.@schemeURI", equalTo("https://ror.org")) + .body("resource.creators.creator[0].affiliation.@affiliationIdentifierScheme", equalTo("ROR")) + // see below for additional fundingReference assertions, which can vary + .body("resource.fundingReferences.fundingReference[0].awardNumber", equalTo("12345")) + .statusCode(OK.getStatusCode()); + + // Out of the box :CVocConf is not set. If you set it with + // https://github.com/gdcc/dataverse-external-vocab-support/blob/de011d239254ff7d651212c565f8604808dcd7e9/examples/config/grantNumberAgencyRor.json + // you can expect different results. + boolean authorsOrcidAndRorEnabled = false; + if (authorsOrcidAndRorEnabled) { + exportDatasetAsDataCite.then().assertThat() + .body("resource.creators.creator[0].affiliation", equalTo("Harvard University")) + // Once https://github.com/IQSS/dataverse/pull/11175 is merged the equalTo bellow + // should be "https://ror.org/03vek6s52" instead of "Harvard University". + .body("resource.creators.creator[0].affiliation.@affiliationIdentifier", equalTo("Harvard University")); + } else { + exportDatasetAsDataCite.then().assertThat() + .body("resource.creators.creator[0].affiliation", equalTo("https://ror.org/03vek6s52")) + .body("resource.creators.creator[0].affiliation.@affiliationIdentifier", equalTo("https://ror.org/03vek6s52")); + } + + // Out of the box :CVocConf is not set. If you set it with + // https://github.com/gdcc/dataverse-external-vocab-support/blob/de011d239254ff7d651212c565f8604808dcd7e9/examples/config/grantNumberAgencyRor.json + // you can expect different results. + boolean grantNumberAgencyRorEnabled = false; + if (grantNumberAgencyRorEnabled) { + exportDatasetAsDataCite.then().assertThat() + .body("resource.fundingReferences.fundingReference[0].funderName", equalTo("National Institute of Health")) + .body("resource.fundingReferences.fundingReference[0].funderIdentifier.@funderIdentifierType", equalTo("ROR")) + .body("resource.fundingReferences.fundingReference[0].funderIdentifier.@schemeURI", equalTo("https://ror.org")) + .body("resource.fundingReferences.fundingReference[0].funderIdentifier", equalTo("https://ror.org/05h1kgg64")); + } else { + exportDatasetAsDataCite.then().assertThat() + .body("resource.fundingReferences.fundingReference[0].funderName", equalTo("https://ror.org/05h1kgg64")) + .body("resource.fundingReferences.fundingReference[0].awardNumber", equalTo("12345")); + } + } + @Test public void getVersionFiles() throws IOException, InterruptedException { Response createUser = UtilIT.createRandomUser(); From 8653513715e4c94f86a9cd3c9bfeabcd1d4ba465 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 21 Jan 2025 15:38:50 -0500 Subject: [PATCH 113/137] note that it take several seconds to load the citation block #11075 --- doc/release-notes/11075-ror.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/11075-ror.md b/doc/release-notes/11075-ror.md index 014ced2dd73..dc8b68512d3 100644 --- a/doc/release-notes/11075-ror.md +++ b/doc/release-notes/11075-ror.md @@ -6,7 +6,7 @@ ROR (Research Organization Registry) has been added as an Author Identifier Type 6\. Update metadata blocks -These changes reflect incremental improvements made to the handling of core metadata fields. +These changes reflect incremental improvements made to the handling of core metadata fields. Expect the loading of the citation block to take several seconds because of its size (especially due to the number of languages). ```shell wget https://raw.githubusercontent.com/IQSS/dataverse/v6.6/scripts/api/data/metadatablocks/citation.tsv From 1fe95d97cfc087ca7f08a68383e3d6adcaa96fa8 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 21 Jan 2025 16:21:16 -0500 Subject: [PATCH 114/137] put ROR regex in line with existing pattern #11075 --- .../harvard/iq/dataverse/ExternalIdentifier.java | 6 ++++-- .../pidproviders/doi/XmlMetadataTemplate.java | 4 ++-- .../dataverse/DatasetFieldValueValidatorTest.java | 2 +- .../doi/datacite/XmlMetadataTemplateTest.java | 15 +++++++-------- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/ExternalIdentifier.java b/src/main/java/edu/harvard/iq/dataverse/ExternalIdentifier.java index 8c4fb6b1325..cf1dcfb3c15 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ExternalIdentifier.java +++ b/src/main/java/edu/harvard/iq/dataverse/ExternalIdentifier.java @@ -13,8 +13,10 @@ public enum ExternalIdentifier { // note: DAI is missing from this list, because it doesn't have resolvable URL ResearcherID("ResearcherID", "https://publons.com/researcher/%s/", "^[A-Z\\d][A-Z\\d-]+[A-Z\\d]$"), ScopusID("ScopusID", "https://www.scopus.com/authid/detail.uri?authorId=%s", "^\\d*$"), - //Requiring ROR to be URL form as we use it where there is no id type field and matching any 9 digit number starting with 0 seems a bit aggressive - ROR("ROR", "https://ror.org/%s", "^(https:\\/\\/ror.org\\/)0[a-hj-km-np-tv-z|0-9]{6}[0-9]{2}$"); + // ROR regex from https://ror.readme.io/docs/identifier + ROR("ROR", "https://ror.org/%s", "^0[a-hj-km-np-tv-z|0-9]{6}[0-9]{2}$"), + // In some contexts, we check for the full ROR URL. + ROR_FULL_URL("ROR", "https://ror.org/%s", "^(https:\\/\\/ror.org\\/)0[a-hj-km-np-tv-z|0-9]{6}[0-9]{2}$"); private String name; private String template; diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 8199b7d9c9f..eb1294a3db7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -626,7 +626,7 @@ private void writeEntityElements(XMLStreamWriter xmlw, String elementName, Strin attributeMap.clear(); boolean isROR=false; String orgName = affiliation; - ExternalIdentifier externalIdentifier = ExternalIdentifier.ROR; + ExternalIdentifier externalIdentifier = ExternalIdentifier.ROR_FULL_URL; if (externalIdentifier.isValidIdentifier(orgName)) { isROR = true; JsonObject jo = getExternalVocabularyValue(orgName); @@ -1528,7 +1528,7 @@ private void writeFundingReferences(XMLStreamWriter xmlw, DvObject dvObject) thr fundingReferenceWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "fundingReferences", fundingReferenceWritten); boolean isROR=false; String funderIdentifier = null; - ExternalIdentifier externalIdentifier = ExternalIdentifier.ROR; + ExternalIdentifier externalIdentifier = ExternalIdentifier.ROR_FULL_URL; if (externalIdentifier.isValidIdentifier(funder)) { isROR = true; JsonObject jo = getExternalVocabularyValue(funder); diff --git a/src/test/java/edu/harvard/iq/dataverse/DatasetFieldValueValidatorTest.java b/src/test/java/edu/harvard/iq/dataverse/DatasetFieldValueValidatorTest.java index da9496e899e..dc04bf7fe01 100644 --- a/src/test/java/edu/harvard/iq/dataverse/DatasetFieldValueValidatorTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/DatasetFieldValueValidatorTest.java @@ -161,7 +161,7 @@ public void testIsValidAuthorIdentifierGnd() { public void testIsValidAuthorIdentifierRor() { DatasetFieldValueValidator validator = new DatasetFieldValueValidator(); Pattern pattern = ExternalIdentifier.valueOf("ROR").getPattern(); - assertTrue(validator.isValidAuthorIdentifier("https://ror.org/03vek6s52", pattern)); + assertTrue(validator.isValidAuthorIdentifier("03vek6s52", pattern)); assertFalse(validator.isValidAuthorIdentifier("junk", pattern)); } diff --git a/src/test/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/XmlMetadataTemplateTest.java b/src/test/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/XmlMetadataTemplateTest.java index 7faa9a0a6fa..2e0d4a12fe0 100644 --- a/src/test/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/XmlMetadataTemplateTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/XmlMetadataTemplateTest.java @@ -194,15 +194,14 @@ public void testDataCiteXMLCreation() throws IOException { assertEquals("https://orcid.org", XmlPath.from(xml).getString("resource.creators.creator[0].nameIdentifier.@schemeURI")); assertEquals("Bob", XmlPath.from(xml).getString("resource.creators.creator[1].creatorName")); assertEquals("Harvard University", XmlPath.from(xml).getString("resource.creators.creator[2].creatorName")); - // FIXME: nameIdentifierScheme, nameIdentifierScheme, and schemeURI should be populated - assertEquals("", XmlPath.from(xml).getString("resource.creators.creator[2].nameIdentifier")); - assertEquals(null, XmlPath.from(xml).getString("resource.creators.creator[2].nameIdentifier.@nameIdentifierScheme")); - assertEquals(null, XmlPath.from(xml).getString("resource.creators.creator[2].nameIdentifier.@schemeURI")); + assertEquals("https://ror.org/03vek6s52", XmlPath.from(xml).getString("resource.creators.creator[2].nameIdentifier")); + assertEquals("ROR", XmlPath.from(xml).getString("resource.creators.creator[2].nameIdentifier.@nameIdentifierScheme")); + assertEquals("https://ror.org", XmlPath.from(xml).getString("resource.creators.creator[2].nameIdentifier.@schemeURI")); assertEquals("Qualitative Data Repository", XmlPath.from(xml).getString("resource.creators.creator[3].creatorName")); - // FIXME: the URL below is not right - assertEquals("https://ror.org/https://ror.org/014trz974", XmlPath.from(xml).getString("resource.creators.creator[3].nameIdentifier")); - assertEquals("ROR", XmlPath.from(xml).getString("resource.creators.creator[3].nameIdentifier.@nameIdentifierScheme")); - assertEquals("https://ror.org", XmlPath.from(xml).getString("resource.creators.creator[3].nameIdentifier.@schemeURI")); + // The nameIdentifier fields below are not populated because the full ROR URL was entered. + assertEquals("", XmlPath.from(xml).getString("resource.creators.creator[3].nameIdentifier")); + assertEquals(null, XmlPath.from(xml).getString("resource.creators.creator[3].nameIdentifier.@nameIdentifierScheme")); + assertEquals(null, XmlPath.from(xml).getString("resource.creators.creator[3].nameIdentifier.@schemeURI")); assertEquals("Dataverse", XmlPath.from(xml).getString("resource.publisher")); } From 7cae923c2d9143006ad8f39ceb0329644cd73643 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20ROUCOU?= Date: Wed, 22 Jan 2025 11:04:47 +0100 Subject: [PATCH 115/137] Fix missing import --- .../java/edu/harvard/iq/dataverse/search/IndexServiceBean.java | 1 + .../edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java | 1 + 2 files changed, 2 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index c4673f79b07..839dd4a7e08 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -16,6 +16,7 @@ import edu.harvard.iq.dataverse.DatasetServiceBean; import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.DatasetVersion.VersionState; +import edu.harvard.iq.dataverse.DatasetVersionFilesServiceBean; import edu.harvard.iq.dataverse.DatasetVersionServiceBean; import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.DataverseLinkingServiceBean; diff --git a/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java index 54b9ed4888a..2b54a4b12cd 100644 --- a/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java @@ -9,6 +9,7 @@ import edu.harvard.iq.dataverse.DatasetFieldType; import edu.harvard.iq.dataverse.DatasetFieldValue; import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.DatasetVersionFilesServiceBean; import edu.harvard.iq.dataverse.DatasetVersionServiceBean; import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.Dataverse.DataverseType; From baa84f578ecea52eb5f66aab57533f3749998030 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 22 Jan 2025 09:33:13 -0500 Subject: [PATCH 116/137] add unit test for ROR_FULL_URL #11075 --- .../iq/dataverse/DatasetFieldValueValidatorTest.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/test/java/edu/harvard/iq/dataverse/DatasetFieldValueValidatorTest.java b/src/test/java/edu/harvard/iq/dataverse/DatasetFieldValueValidatorTest.java index dc04bf7fe01..7320cf7acfe 100644 --- a/src/test/java/edu/harvard/iq/dataverse/DatasetFieldValueValidatorTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/DatasetFieldValueValidatorTest.java @@ -165,6 +165,14 @@ public void testIsValidAuthorIdentifierRor() { assertFalse(validator.isValidAuthorIdentifier("junk", pattern)); } + @Test + public void testIsValidAuthorIdentifierRorFull() { + DatasetFieldValueValidator validator = new DatasetFieldValueValidator(); + Pattern pattern = ExternalIdentifier.valueOf("ROR_FULL_URL").getPattern(); + assertTrue(validator.isValidAuthorIdentifier("https://ror.org/03vek6s52", pattern)); + assertFalse(validator.isValidAuthorIdentifier("junk", pattern)); + } + final Validator validator = Validation.buildDefaultValidatorFactory().getValidator(); @ParameterizedTest From 308ea2ac3577cd0cdf4c3c0bd90d66aa6cf4885f Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 22 Jan 2025 09:36:26 -0500 Subject: [PATCH 117/137] use correct NIH #11075 --- .../java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 7f2569982f3..c1273fba9fc 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -4397,7 +4397,7 @@ public void testDataCiteExport() throws IOException { .add(Json.createObjectBuilder() .add("authorName", Json.createObjectBuilder() - .add("value", "https://ror.org/05h1kgg64") // NIH + .add("value", "https://ror.org/01cwqze88") // NIH .add("typeClass", "primitive") .add("multiple", false) .add("typeName", "grantNumberAgency") @@ -4464,13 +4464,13 @@ public void testDataCiteExport() throws IOException { boolean grantNumberAgencyRorEnabled = false; if (grantNumberAgencyRorEnabled) { exportDatasetAsDataCite.then().assertThat() - .body("resource.fundingReferences.fundingReference[0].funderName", equalTo("National Institute of Health")) + .body("resource.fundingReferences.fundingReference[0].funderName", equalTo("National Institutes of Health")) .body("resource.fundingReferences.fundingReference[0].funderIdentifier.@funderIdentifierType", equalTo("ROR")) .body("resource.fundingReferences.fundingReference[0].funderIdentifier.@schemeURI", equalTo("https://ror.org")) - .body("resource.fundingReferences.fundingReference[0].funderIdentifier", equalTo("https://ror.org/05h1kgg64")); + .body("resource.fundingReferences.fundingReference[0].funderIdentifier", equalTo("https://ror.org/01cwqze88")); } else { exportDatasetAsDataCite.then().assertThat() - .body("resource.fundingReferences.fundingReference[0].funderName", equalTo("https://ror.org/05h1kgg64")) + .body("resource.fundingReferences.fundingReference[0].funderName", equalTo("https://ror.org/01cwqze88")) .body("resource.fundingReferences.fundingReference[0].awardNumber", equalTo("12345")); } } From b9d0146b07c9c88e0b5b8759fe2cff10df19029d Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 22 Jan 2025 10:28:35 -0500 Subject: [PATCH 118/137] add to user guide how to enter an author identifier #11075 --- doc/sphinx-guides/source/user/dataset-management.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/user/dataset-management.rst b/doc/sphinx-guides/source/user/dataset-management.rst index b3a14554b40..6b2f48fb8d9 100755 --- a/doc/sphinx-guides/source/user/dataset-management.rst +++ b/doc/sphinx-guides/source/user/dataset-management.rst @@ -50,8 +50,10 @@ Adding a New Dataset #. Navigate to the Dataverse collection in which you want to add a dataset. #. Click on the "Add Data" button and select "New Dataset" in the dropdown menu. **Note:** If you are on the root Dataverse collection, your My Data page or click the "Add Data" link in the navbar, the dataset you create will be hosted in the root Dataverse collection. You can change this by selecting another Dataverse collection you have proper permissions to create datasets in, from the Host Dataverse collection dropdown in the create dataset form. This option to choose will not be available after you create the dataset. -#. To quickly get started, enter at minimum all the required fields with an asterisk (e.g., the Dataset Title, Author Name, - Description Text, Point of Contact Email, and Subject) to get a Data Citation with a DOI. +#. To quickly get started, enter at minimum all the required fields with an asterisk (e.g., the Dataset Title, Author Name, Description Text, Point of Contact Email, and Subject) to get a Data Citation with a DOI. + + #. When entering author identifiers, select the type from the dropdown (e.g. "ORCID") and under "Identifier" enter just the unique identifier (e.g. "0000-0002-1825-0097") rather than the full URL (e.g. "https://orcid.org/0000-0002-1825-0097"). + #. Scroll down to the "Files" section and click on "Select Files to Add" to add all the relevant files to your Dataset. You can also upload your files directly from your Dropbox. **Tip:** You can drag and drop or select multiple files at a time from your desktop directly into the upload widget. Your files will appear below the "Select Files to Add" button where you can add a From d94a7a243a4657a5486b8d052d0aa37fdff4631b Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 22 Jan 2025 10:33:15 -0500 Subject: [PATCH 119/137] explain how to add author identifier in release note #11075 --- doc/release-notes/11075-ror.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/11075-ror.md b/doc/release-notes/11075-ror.md index dc8b68512d3..de32eebccdf 100644 --- a/doc/release-notes/11075-ror.md +++ b/doc/release-notes/11075-ror.md @@ -1,6 +1,6 @@ ### ROR (Research Organization Registry) as Author Identifier Type -ROR (Research Organization Registry) has been added as an Author Identifier Type (alongside ORCID, etc.) for when the author is an organization rather than a person. See , #11075, and #11118. +ROR (Research Organization Registry) has been added as an Author Identifier Type (alongside ORCID, etc.) for when the author is an organization rather than a person. As with all author identifiers, be sure to select the proper identifier type (e.g. "ROR") and enter just the unique identifier (e.g. "03vek6s52") rather than the full URL (e.g. "https://ror.org/03vek6s52"). Like ORCID, ROR will appear in the "Datacite" metadata export format. See also the [ROR](https://ror.org) website, a new [note](https://dataverse-guide--11118.org.readthedocs.build/en/11118/user/dataset-management.html#adding-a-new-dataset) in a guides about entering author identifiers, #11075, and #11118. ## Upgrade Instructions From 8d03b213694fc1ad56242dbad75da1cfe1d96459 Mon Sep 17 00:00:00 2001 From: stevenferey Date: Wed, 22 Jan 2025 17:21:29 +0100 Subject: [PATCH 120/137] =?UTF-8?q?=E2=80=8BLabel=20adaptation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/propertyFiles/Bundle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index a87bbc760fc..89a6e61d3d8 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -544,7 +544,7 @@ dashboard.card.metadataexport.header=Metadata Export dashboard.card.metadataexport.message=Dataset metadata export is only available through the {0} API. Learn more in the {0} {1}API Guide{2}. dashboard.card.move.data=Data dashboard.card.move.dataset.manage=Move Dataset -dashboard.card.move.dataverse.manage=Move Collection +dashboard.card.move.dataverse.manage=Move Dataverse #harvestclients.xhtml harvestclients.title=Manage Harvesting Clients From bff877b022a7d885e06e042d1ca20585b664ccac Mon Sep 17 00:00:00 2001 From: stevenferey Date: Wed, 22 Jan 2025 17:27:57 +0100 Subject: [PATCH 121/137] =?UTF-8?q?=E2=80=8BAdaptation=20of=20the=20error?= =?UTF-8?q?=20message=20in=20case=20of=20forced=20collection=20move=20requ?= =?UTF-8?q?ired?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/propertyFiles/Bundle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 89a6e61d3d8..0ba6dda70b7 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -2805,7 +2805,7 @@ dataverses.api.delete.featured.collections.successful=Featured dataverses have b dataverses.api.move.dataverse.error.metadataBlock=Dataverse metadata block is not in target dataverse. dataverses.api.move.dataverse.error.dataverseLink=Dataverse is linked to target dataverse or one of its parents. dataverses.api.move.dataverse.error.datasetLink=Dataset is linked to target dataverse or one of its parents. -dataverses.api.move.dataverse.error.forceMove=Please use the parameter ?forceMove=true to complete the move. This will remove anything from the dataverse that is not compatible with the target dataverse. +dataverses.api.move.dataverse.error.forceMove=Please use the API and see "Move a Dataverse Collection" with the parameter ?forceMove=true to complete the move. This will remove anything from the dataverse that is not compatible with the target dataverse. dataverses.api.create.dataset.error.mustIncludeVersion=Please provide initial version in the dataset json dataverses.api.create.dataset.error.superuserFiles=Only a superuser may add files via this api dataverses.api.create.dataset.error.mustIncludeAuthorName=Please provide author name in the dataset json From acda4587df7cf7fa176be8adbfb5070c467bf074 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 22 Jan 2025 13:57:12 -0500 Subject: [PATCH 122/137] bump sql script version #8739 --- src/main/resources/db/migration/{V6.5.0.2.sql => V6.5.0.3.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V6.5.0.2.sql => V6.5.0.3.sql} (100%) diff --git a/src/main/resources/db/migration/V6.5.0.2.sql b/src/main/resources/db/migration/V6.5.0.3.sql similarity index 100% rename from src/main/resources/db/migration/V6.5.0.2.sql rename to src/main/resources/db/migration/V6.5.0.3.sql From e6b5d42082f03b090cb9f7220f5b7bae5384f9e4 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 22 Jan 2025 14:01:32 -0500 Subject: [PATCH 123/137] Update doc/sphinx-guides/source/developers/tips.rst Co-authored-by: Omer Fahim --- doc/sphinx-guides/source/developers/tips.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/tips.rst b/doc/sphinx-guides/source/developers/tips.rst index a0e92015cbd..8d715c7d016 100755 --- a/doc/sphinx-guides/source/developers/tips.rst +++ b/doc/sphinx-guides/source/developers/tips.rst @@ -185,7 +185,7 @@ Solr Once some Dataverse collections, datasets, and files have been created and indexed, you can experiment with searches directly from Solr at http://localhost:8983/solr/#/collection1/query and look at the JSON output of searches, such as this wildcard search: http://localhost:8983/solr/collection1/select?q=*%3A*&wt=json&indent=true . You can also get JSON output of static fields Solr knows about: http://localhost:8983/solr/collection1/schema/fields -You can simply double-click "start.jar" rather that running ``java -jar start.jar`` from the command line. Figuring out how to stop Solr after double-clicking it is an exercise for the reader. +You can simply double-click "start.jar" rather than running ``java -jar start.jar`` from the command line. Figuring out how to stop Solr after double-clicking it is an exercise for the reader. .. _update-solr-schema-dev: From 8f697d2905ec19ee6704f0ba4ada2d975f01d238 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 23 Jan 2025 14:36:22 -0500 Subject: [PATCH 124/137] clarify "making releases" process based on 6.5 release experience (#11092) * make ansible PR before main PR * clarify about bumping VERSION for guides * skipping code review ok for "merge develop to master" * put the base.image.version PR through code review * when to lift the code freeze * clarify next release and milestones * announce releases on Zulip * edit tab on board that shows milestone * add "dedicated issue" label * change title to deploy to internal * no issue for small steps * how to upgrade demo Co-authored-by: Omer Fahim * Update doc/sphinx-guides/source/developers/making-releases.rst * Update doc/sphinx-guides/source/developers/making-releases.rst * Update doc/sphinx-guides/source/developers/making-releases.rst * Update doc/sphinx-guides/source/developers/making-releases.rst * remove trailing newline --------- Co-authored-by: Omer Fahim --- .../source/developers/making-releases.rst | 75 ++++++++++++++++--- 1 file changed, 63 insertions(+), 12 deletions(-) diff --git a/doc/sphinx-guides/source/developers/making-releases.rst b/doc/sphinx-guides/source/developers/making-releases.rst index aed174f60d4..8f9b43eabcb 100755 --- a/doc/sphinx-guides/source/developers/making-releases.rst +++ b/doc/sphinx-guides/source/developers/making-releases.rst @@ -30,7 +30,24 @@ Early on, make sure it's clear what type of release this is. The steps below des Ensure Issues Have Been Created ------------------------------- -In advance of a release, GitHub issues should have been created already that capture certain steps. See https://github.com/IQSS/dataverse-pm/issues/335 for examples. +Some of the steps in this document are well-served by having their own dedicated GitHub issue. You'll see a label like this on them: + +|dedicated| + +There are a variety of reasons why a step might deserve its own dedicated issue: + +- The step can be done by a team member other than the person doing the release. +- Stakeholders might be interested in the status of a step (e.g. has the released been deployed to the demo site). + +Steps don't get their own dedicated issue if it would be confusing to have multiple people involved. Too many cooks in the kitchen, as they say. Also, some steps are so small the overhead of an issue isn't worth it. + +Before the release even begins you can coordinate with the project manager about the creation of these issues. + +.. |dedicated| raw:: html + + + Dedicated Issue +   Declare a Code Freeze --------------------- @@ -40,18 +57,25 @@ The following steps are made more difficult if code is changing in the "develop" Conduct Performance Testing --------------------------- +|dedicated| + See :doc:`/qa/performance-tests` for details. -Conduct Smoke Testing ---------------------- +Conduct Regression Testing +--------------------------- + +|dedicated| See :doc:`/qa/testing-approach` for details. +Refer to the provided regression checklist for the list of items to verify during the testing process: `Regression Checklist `_. .. _write-release-notes: Write Release Notes ------------------- +|dedicated| + Developers express the need for an addition to release notes by creating a "release note snippet" in ``/doc/release-notes`` containing the name of the issue they're working on. The name of the branch could be used for the filename with ".md" appended (release notes are written in Markdown) such as ``5053-apis-custom-homepage.md``. See :ref:`writing-release-note-snippets` for how this is described for contributors. The task at or near release time is to collect these snippets into a single file. @@ -62,17 +86,22 @@ The task at or near release time is to collect these snippets into a single file - Include instructions describing the steps required to upgrade the application from the previous version. These must be customized for release numbers and special circumstances such as changes to metadata blocks and infrastructure. - Take the release notes .md through the regular Code Review and QA process. That is, make a pull request. Here's an example: https://github.com/IQSS/dataverse/pull/10866 -Upgrade Instructions for Internal ---------------------------------- +Deploy Release Candidate to Internal +------------------------------------ + +|dedicated| To upgrade internal, go to /doc/release-notes, open the release-notes.md file for the current release and perform all the steps under "Upgrade Instructions". Deploy Release Candidate to Demo -------------------------------- +|dedicated| + First, build the release candidate. ssh into the dataverse-internal server and undeploy the current war file. +Go to /doc/release-notes, open the release-notes.md file for the current release, and perform all the steps under "Upgrade Instructions". Go to https://jenkins.dataverse.org/job/IQSS_Dataverse_Internal/ and make the following adjustments to the config: @@ -91,6 +120,8 @@ ssh into the demo server and follow the upgrade instructions in the release note Prepare Release Branch ---------------------- +|dedicated| + The release branch will have the final changes such as bumping the version number. Usually we branch from the "develop" branch to create the release branch. If we are creating a hotfix for a particular version (5.11, for example), we branch from the tag (e.g. ``v5.11``). @@ -116,18 +147,20 @@ Return to the parent pom and make the following change, which is necessary for p (Before you make this change the value should be ``${parsedVersion.majorVersion}.${parsedVersion.nextMinorVersion}``. Later on, after cutting a release, we'll change it back to that value.) -For a regular release, make the changes above in the release branch you created, make a pull request, and merge it into the "develop" branch. Like usual, you can safely delete the branch after the merge is complete. +For a regular release, make the changes above in the release branch you created, but hold off for a moment on making a pull requests because Jenkins will fail because it will be testing the previous release. -If you are making a hotfix release, make the pull request against the "master" branch. Do not delete the branch after merging because we will later merge it into the "develop" branch to pick up the hotfix. More on this later. +In the dataverse-ansible repo make bump the version in `jenkins.yml `_ and make a pull request such as https://github.com/gdcc/dataverse-ansible/pull/386. Wait for it to be merged. Note that bumping on the Jenkins side like this will mean that all pull requests will show failures in Jenkins until they are updated to the version we are releasing. -Either way, as usual, you should ensure that all tests are passing. Please note that you will need to bump the version in `jenkins.yml `_ in dataverse-ansible to get the tests to pass. Consider doing this before making the pull request. Alternatively, you can bump jenkins.yml after making the pull request and re-run the Jenkins job to make sure tests pass. +Once dataverse-ansible has been merged, return to the branch you created above ("10852-bump-to-6.4" or whatever) and make a pull request. Ensure that all tests are passing and then put the PR through the normal review and QA process. + +If you are making a hotfix release, make the pull request against the "master" branch. Do not delete the branch after merging because we will later merge it into the "develop" branch to pick up the hotfix. More on this later. Merge "develop" into "master" ----------------------------- If this is a regular (non-hotfix) release, create a pull request to merge the "develop" branch into the "master" branch using this "compare" link: https://github.com/IQSS/dataverse/compare/master...develop -Once important tests have passed (compile, unit tests, etc.), merge the pull request. Don't worry about style tests failing such as for shell scripts. +Once important tests have passed (compile, unit tests, etc.), merge the pull request (skipping code review is ok). Don't worry about style tests failing such as for shell scripts. If this is a hotfix release, skip this whole "merge develop to master" step (the "develop" branch is not involved until later). @@ -160,7 +193,7 @@ Go to https://jenkins.dataverse.org/job/guides.dataverse.org/ and make the follo - Repository URL: ``https://github.com/IQSS/dataverse.git`` - Branch Specifier (blank for 'any'): ``*/master`` -- ``VERSION`` (under "Build Steps"): ``5.10.1`` (for example) +- ``VERSION`` (under "Build Steps"): bump to the next release. Don't prepend a "v". Use ``5.10.1`` (for example) Click "Save" then "Build Now". @@ -265,24 +298,37 @@ Close Milestone on GitHub and Create a New One You can find our milestones at https://github.com/IQSS/dataverse/milestones -Now that we've published the release, close the milestone and create a new one. +Now that we've published the release, close the milestone and create a new one for the **next** release, the release **after** the one we're working on, that is. Note that for milestones we use just the number without the "v" (e.g. "5.10.1"). +On the project board at https://github.com/orgs/IQSS/projects/34 edit the tab (view) that shows the milestone to show the next milestone. + Update the Container Base Image Version Property ------------------------------------------------ +|dedicated| + Create a new branch (any name is fine but ``prepare-next-iteration`` is suggested) and update the following files to prepare for the next development cycle: - modules/dataverse-parent/pom.xml -> ```` -> profile "ct" -> ```` -> Set ```` to ``${parsedVersion.majorVersion}.${parsedVersion.nextMinorVersion}`` -Now create a pull request and merge it. +Create a pull request and put it through code review, like usual. Give it a milestone of the next release, the one **after** the one we're working on. Once the pull request has been approved, merge it. It should the the first PR merged of the next release. For more background, see :ref:`base-supported-image-tags`. For an example, see https://github.com/IQSS/dataverse/pull/10896 +Lift the Code Freeze and Encourage Developers to Update Their Branches +---------------------------------------------------------------------- + +It's now safe to lift the code freeze. We can start merging pull requests into the "develop" branch for the next release. + +Let developers know that they should merge the latest from the "develop" branch into any branches they are working on. + Deploy Final Release on Demo ---------------------------- +|dedicated| + Above you already did the hard work of deploying a release candidate to https://demo.dataverse.org. It should be relatively straightforward to undeploy the release candidate and deploy the final release. Update SchemaSpy @@ -316,6 +362,11 @@ Announce the Release on the Mailing List Post a message at https://groups.google.com/g/dataverse-community +Announce the Release on Zulip +----------------------------- + +Post a message under #community at https://dataverse.zulipchat.com + For Hotfixes, Merge Hotfix Branch into "develop" and Rename SQL Scripts ----------------------------------------------------------------------- From 69de6dd3fd972d98cc557e32f46e27d091378232 Mon Sep 17 00:00:00 2001 From: GPortas Date: Fri, 24 Jan 2025 12:45:03 +0000 Subject: [PATCH 125/137] Fixed: DataverseFeaturedItemsIT --- .../harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java index 2fa45200977..032c1739d53 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItemsIT.java @@ -28,7 +28,7 @@ public void testDeleteFeaturedItem() { // Should return not found when passing incorrect item id Response deleteFeatureItemResponse = UtilIT.deleteDataverseFeaturedItem(100000L, apiToken); deleteFeatureItemResponse.then() - .body("message", equalTo(MessageFormat.format(BundleUtil.getStringFromBundle("dataverseFeaturedItems.errors.notFound"), featuredItemId))) + .body("message", equalTo(MessageFormat.format(BundleUtil.getStringFromBundle("dataverseFeaturedItems.errors.notFound"), 100000L))) .assertThat().statusCode(NOT_FOUND.getStatusCode()); // Should return unauthorized when passing correct id and user does not have permissions @@ -57,7 +57,7 @@ public void testUpdateFeaturedItem() { // Should return not found when passing incorrect item id Response updateFeatureItemResponse = UtilIT.updateDataverseFeaturedItem(100000L, "updatedTitle", 1, false, null, apiToken); updateFeatureItemResponse.then() - .body("message", equalTo(MessageFormat.format(BundleUtil.getStringFromBundle("dataverseFeaturedItems.errors.notFound"), featuredItemId))) + .body("message", equalTo(MessageFormat.format(BundleUtil.getStringFromBundle("dataverseFeaturedItems.errors.notFound"), 100000L))) .assertThat().statusCode(NOT_FOUND.getStatusCode()); // Should return unauthorized when passing correct id and user does not have permissions From 063fa5effe7b78b374be406a0ff5ab285e13df02 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 24 Jan 2025 14:53:28 -0500 Subject: [PATCH 126/137] Apply suggestions from code review Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/api/auth.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/api/auth.rst b/doc/sphinx-guides/source/api/auth.rst index 5acf0fb0ff7..8dffb914e29 100644 --- a/doc/sphinx-guides/source/api/auth.rst +++ b/doc/sphinx-guides/source/api/auth.rst @@ -94,13 +94,13 @@ By default, the Bearer token is expected to include the following claims that wi - ``lastName`` - ``emailAddress`` -The one parameter required by default is ``termsAccepted``` which must be set to true, indicating that the user has seen and accepted the Terms of Use of the Installation. +The one parameter required by default is ``termsAccepted`` which must be set to true, indicating that the user has seen and accepted the Terms of Use of the installation. -If the feature flag ``api-bearer-auth-handle-tos-acceptance-in-idp``` is enabled (along with the ``api-bearer-auth`` feature flag), Dataverse assumes that the Terms of Service acceptance was handled by the identity provider, e.g. in the OIDC ``consent``` dialog, and the `termsAccepted``` parameter is not needed. +If the feature flag ``api-bearer-auth-handle-tos-acceptance-in-idp`` is enabled (along with the ``api-bearer-auth`` feature flag), Dataverse assumes that the Terms of Service acceptance was handled by the identity provider, e.g. in the OIDC ``consent`` dialog, and the ``termsAccepted`` parameter is not needed. There is another flag called ``api-bearer-auth-provide-missing-claims`` that can be enabled (along with the ``api-bearer-auth`` feature flag) to allow sending missing user claims in the registration JSON. This is useful when the identity provider does not supply the necessary claims listed above. If properties are provided in the JSON, but corresponding claims already exist in the identity provider, an error will be thrown, outlining the conflicting properties. Note that supplying missing claims is configured via a separate feature flag because using it may introduce user impersonation issues, for example if the identity provider does not provide an email field and the user submits an email address they do not own. -In all cases, the submitted JSON can optionally include the fields ``position`` or ``affiliation``, which will be added to the users's Dataverse account profile. These fields are optional, and if not provided, they will be persisted as empty in Dataverse. +In all cases, the submitted JSON can optionally include the fields ``position`` or ``affiliation``, which will be added to the user's Dataverse account. These fields are optional, and if not provided, they will be persisted as empty in Dataverse. Signed URLs ----------- From e14476706840283d74ee5f6a85fd693e0dcb4d02 Mon Sep 17 00:00:00 2001 From: GPortas Date: Mon, 27 Jan 2025 14:40:40 +0000 Subject: [PATCH 127/137] Fixed: wrong endpoint path name --- .../java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java | 2 +- src/main/java/edu/harvard/iq/dataverse/api/Access.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java index ee91c258426..852baf5b0db 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java @@ -80,7 +80,7 @@ public void setImageFileName(String imageFileName) { public String getImageFileUrl() { if (id != null && imageFileName != null) { - return SystemConfig.getDataverseSiteUrlStatic() + "/api/access/dataverseFeatureItemImage/" + id; + return SystemConfig.getDataverseSiteUrlStatic() + "/api/access/dataverseFeaturedItemImage/" + id; } return null; } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index 0bf02411d3a..c819da6ffe8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -1986,7 +1986,7 @@ private URI handleCustomZipDownload(User user, String customZipServiceUrl, Strin @GET @AuthRequired @Produces({"image/png"}) - @Path("dataverseFeatureItemImage/{itemId}") + @Path("dataverseFeaturedItemImage/{itemId}") public InputStream getDataverseFeatureItemImage(@Context ContainerRequestContext crc, @PathParam("itemId") Long itemId) { DataverseFeaturedItem dataverseFeaturedItem; try { From fb61d89a88d7d2d7d8eeb2054509b34ad26301a3 Mon Sep 17 00:00:00 2001 From: GPortas Date: Mon, 27 Jan 2025 14:50:49 +0000 Subject: [PATCH 128/137] Added: docs for api/access/dataverseFeaturedItemImage/{ID} --- doc/sphinx-guides/source/api/native-api.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 9fb7341044d..249da446c8c 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1344,6 +1344,25 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/dataverseFeaturedItems/1" +Get A Collection Featured Item Image +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Returns the image of a featured item if one is assigned, given the featured item ``id``: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=1 + + curl -H "X-Dataverse-key:$API_TOKEN" -X GET "$SERVER_URL/api/access/dataverseFeaturedItemImage/{ID}" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X GET "https://demo.dataverse.org/api/access/dataverseFeaturedItemImage/1" + Datasets -------- From e91524bc8362a9b314f48ffb5dd16f13a924616f Mon Sep 17 00:00:00 2001 From: GPortas Date: Mon, 27 Jan 2025 15:01:10 +0000 Subject: [PATCH 129/137] Refactor: package structure for dataverse featured items --- src/main/java/edu/harvard/iq/dataverse/Dataverse.java | 1 + .../java/edu/harvard/iq/dataverse/EjbDataverseEngine.java | 1 + src/main/java/edu/harvard/iq/dataverse/api/Access.java | 2 ++ .../edu/harvard/iq/dataverse/api/DataverseFeaturedItems.java | 3 ++- src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java | 2 ++ .../{ => dataverse/featured}/DataverseFeaturedItem.java | 3 ++- .../featured}/DataverseFeaturedItemServiceBean.java | 3 ++- .../harvard/iq/dataverse/engine/command/CommandContext.java | 1 + .../impl/AbstractWriteDataverseFeaturedItemCommand.java | 2 ++ .../command/impl/CreateDataverseFeaturedItemCommand.java | 2 +- .../dataverse/engine/command/impl/DeleteDataverseCommand.java | 2 +- .../command/impl/DeleteDataverseFeaturedItemCommand.java | 2 +- .../engine/command/impl/GetDataverseFeaturedItemCommand.java | 2 +- .../command/impl/ListDataverseFeaturedItemsCommand.java | 2 +- .../command/impl/UpdateDataverseFeaturedItemCommand.java | 2 +- .../command/impl/UpdateDataverseFeaturedItemsCommand.java | 2 +- .../java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java | 1 + .../edu/harvard/iq/dataverse/engine/TestCommandContext.java | 1 + .../command/impl/CreateDataverseFeaturedItemCommandTest.java | 4 ++-- 19 files changed, 26 insertions(+), 12 deletions(-) rename src/main/java/edu/harvard/iq/dataverse/{ => dataverse/featured}/DataverseFeaturedItem.java (95%) rename src/main/java/edu/harvard/iq/dataverse/{ => dataverse/featured}/DataverseFeaturedItemServiceBean.java (97%) diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java index f73405222b8..9bb8992e789 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java +++ b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java @@ -1,5 +1,6 @@ package edu.harvard.iq.dataverse; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItem; import edu.harvard.iq.dataverse.harvest.client.HarvestingClient; import edu.harvard.iq.dataverse.authorization.DataverseRole; import edu.harvard.iq.dataverse.search.savedsearch.SavedSearch; diff --git a/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java b/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java index f8ba218d485..0f211dc6713 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java +++ b/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java @@ -4,6 +4,7 @@ import edu.harvard.iq.dataverse.actionlogging.ActionLogServiceBean; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUserServiceBean; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItemServiceBean; import edu.harvard.iq.dataverse.util.cache.CacheFactoryBean; import edu.harvard.iq.dataverse.engine.DataverseEngine; import edu.harvard.iq.dataverse.authorization.Permission; diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index c819da6ffe8..2a27c89eaaa 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -27,6 +27,8 @@ import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter; import edu.harvard.iq.dataverse.datavariable.DataVariable; import edu.harvard.iq.dataverse.datavariable.VariableServiceBean; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItem; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItemServiceBean; import edu.harvard.iq.dataverse.engine.command.Command; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItems.java b/src/main/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItems.java index fdaccaa08d2..a77ea000415 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItems.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItems.java @@ -1,8 +1,9 @@ package edu.harvard.iq.dataverse.api; -import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.api.auth.AuthRequired; import edu.harvard.iq.dataverse.api.dto.UpdatedDataverseFeaturedItemDTO; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItem; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItemServiceBean; import edu.harvard.iq.dataverse.engine.command.impl.*; import edu.harvard.iq.dataverse.util.BundleUtil; import jakarta.ejb.Stateless; diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 8cfe9b9bcf6..bbd9476b9e8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -18,6 +18,8 @@ import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.authorization.users.User; import edu.harvard.iq.dataverse.dataverse.DataverseUtil; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItem; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItemServiceBean; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.impl.*; import edu.harvard.iq.dataverse.pidproviders.PidProvider; diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java b/src/main/java/edu/harvard/iq/dataverse/dataverse/featured/DataverseFeaturedItem.java similarity index 95% rename from src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java rename to src/main/java/edu/harvard/iq/dataverse/dataverse/featured/DataverseFeaturedItem.java index 852baf5b0db..53d09516789 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItem.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataverse/featured/DataverseFeaturedItem.java @@ -1,5 +1,6 @@ -package edu.harvard.iq.dataverse; +package edu.harvard.iq.dataverse.dataverse.featured; +import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.util.SystemConfig; import jakarta.persistence.*; import jakarta.validation.constraints.NotBlank; diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/dataverse/featured/DataverseFeaturedItemServiceBean.java similarity index 97% rename from src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java rename to src/main/java/edu/harvard/iq/dataverse/dataverse/featured/DataverseFeaturedItemServiceBean.java index 89a970fae64..56cdaf5692e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedItemServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataverse/featured/DataverseFeaturedItemServiceBean.java @@ -1,5 +1,6 @@ -package edu.harvard.iq.dataverse; +package edu.harvard.iq.dataverse.dataverse.featured; +import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.FileUtil; diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/CommandContext.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/CommandContext.java index b58f5f07ebc..42f2616cd80 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/CommandContext.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/CommandContext.java @@ -2,6 +2,7 @@ import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUserServiceBean; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItemServiceBean; import edu.harvard.iq.dataverse.search.IndexServiceBean; import edu.harvard.iq.dataverse.search.SearchServiceBean; import edu.harvard.iq.dataverse.actionlogging.ActionLogServiceBean; diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseFeaturedItemCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseFeaturedItemCommand.java index ae2db1c752a..8c4a8281345 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseFeaturedItemCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseFeaturedItemCommand.java @@ -2,6 +2,8 @@ import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItem; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItemServiceBean; import edu.harvard.iq.dataverse.engine.command.AbstractCommand; import edu.harvard.iq.dataverse.engine.command.CommandContext; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java index b21ed1403cd..24732d05c8b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java @@ -1,7 +1,7 @@ package edu.harvard.iq.dataverse.engine.command.impl; import edu.harvard.iq.dataverse.Dataverse; -import edu.harvard.iq.dataverse.DataverseFeaturedItem; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItem; import edu.harvard.iq.dataverse.api.dto.NewDataverseFeaturedItemDTO; import edu.harvard.iq.dataverse.engine.command.CommandContext; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataverseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataverseCommand.java index 7fbb659ab96..84a0ab0f3f2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataverseCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataverseCommand.java @@ -1,7 +1,7 @@ package edu.harvard.iq.dataverse.engine.command.impl; import edu.harvard.iq.dataverse.Dataverse; -import edu.harvard.iq.dataverse.DataverseFeaturedItem; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItem; import edu.harvard.iq.dataverse.DataverseFieldTypeInputLevel; import edu.harvard.iq.dataverse.authorization.DataverseRole; import edu.harvard.iq.dataverse.RoleAssignment; diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataverseFeaturedItemCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataverseFeaturedItemCommand.java index 3a7504b8c77..215863a44da 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataverseFeaturedItemCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataverseFeaturedItemCommand.java @@ -1,7 +1,7 @@ package edu.harvard.iq.dataverse.engine.command.impl; import edu.harvard.iq.dataverse.Dataverse; -import edu.harvard.iq.dataverse.DataverseFeaturedItem; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItem; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.engine.command.*; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDataverseFeaturedItemCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDataverseFeaturedItemCommand.java index 722c8050dbd..c594887b6ed 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDataverseFeaturedItemCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDataverseFeaturedItemCommand.java @@ -1,6 +1,6 @@ package edu.harvard.iq.dataverse.engine.command.impl; -import edu.harvard.iq.dataverse.DataverseFeaturedItem; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItem; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.engine.command.AbstractCommand; import edu.harvard.iq.dataverse.engine.command.CommandContext; diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListDataverseFeaturedItemsCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListDataverseFeaturedItemsCommand.java index 474747ccb0d..0d4051fc7d5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListDataverseFeaturedItemsCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListDataverseFeaturedItemsCommand.java @@ -1,7 +1,7 @@ package edu.harvard.iq.dataverse.engine.command.impl; import edu.harvard.iq.dataverse.Dataverse; -import edu.harvard.iq.dataverse.DataverseFeaturedItem; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItem; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.engine.command.AbstractCommand; import edu.harvard.iq.dataverse.engine.command.CommandContext; diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemCommand.java index a54f0324a3e..ed6fe825b03 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemCommand.java @@ -1,7 +1,7 @@ package edu.harvard.iq.dataverse.engine.command.impl; import edu.harvard.iq.dataverse.Dataverse; -import edu.harvard.iq.dataverse.DataverseFeaturedItem; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItem; import edu.harvard.iq.dataverse.api.dto.UpdatedDataverseFeaturedItemDTO; import edu.harvard.iq.dataverse.engine.command.*; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemsCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemsCommand.java index 12a972f2e47..0368efef6b0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemsCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemsCommand.java @@ -1,7 +1,7 @@ package edu.harvard.iq.dataverse.engine.command.impl; import edu.harvard.iq.dataverse.Dataverse; -import edu.harvard.iq.dataverse.DataverseFeaturedItem; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItem; import edu.harvard.iq.dataverse.api.dto.NewDataverseFeaturedItemDTO; import edu.harvard.iq.dataverse.api.dto.UpdatedDataverseFeaturedItemDTO; import edu.harvard.iq.dataverse.authorization.Permission; diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 1f27eef8e7f..b88dfaef4b5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -25,6 +25,7 @@ import edu.harvard.iq.dataverse.datavariable.VariableCategory; import edu.harvard.iq.dataverse.datavariable.VariableMetadata; import edu.harvard.iq.dataverse.datavariable.VariableRange; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItem; import edu.harvard.iq.dataverse.license.License; import edu.harvard.iq.dataverse.globus.FileDetailsHolder; import edu.harvard.iq.dataverse.harvest.client.HarvestingClient; diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/TestCommandContext.java b/src/test/java/edu/harvard/iq/dataverse/engine/TestCommandContext.java index fd7e3f69bd2..4fec86c5b0b 100644 --- a/src/test/java/edu/harvard/iq/dataverse/engine/TestCommandContext.java +++ b/src/test/java/edu/harvard/iq/dataverse/engine/TestCommandContext.java @@ -9,6 +9,7 @@ import edu.harvard.iq.dataverse.confirmemail.ConfirmEmailServiceBean; import edu.harvard.iq.dataverse.datacapturemodule.DataCaptureModuleServiceBean; import edu.harvard.iq.dataverse.dataset.DatasetTypeServiceBean; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItemServiceBean; import edu.harvard.iq.dataverse.engine.command.Command; import edu.harvard.iq.dataverse.engine.command.CommandContext; import edu.harvard.iq.dataverse.ingest.IngestServiceBean; diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommandTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommandTest.java index 01c5f499b0f..f31299526bd 100644 --- a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommandTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommandTest.java @@ -1,8 +1,8 @@ package edu.harvard.iq.dataverse.engine.command.impl; import edu.harvard.iq.dataverse.Dataverse; -import edu.harvard.iq.dataverse.DataverseFeaturedItem; -import edu.harvard.iq.dataverse.DataverseFeaturedItemServiceBean; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItem; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItemServiceBean; import edu.harvard.iq.dataverse.api.dto.NewDataverseFeaturedItemDTO; import edu.harvard.iq.dataverse.engine.command.CommandContext; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; From a986a674da6ce91128a67feea74b4bb8a701475b Mon Sep 17 00:00:00 2001 From: GPortas Date: Mon, 27 Jan 2025 15:24:02 +0000 Subject: [PATCH 130/137] Added: new settings description to #10943 release notes --- doc/release-notes/10943-featured-items.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/release-notes/10943-featured-items.md b/doc/release-notes/10943-featured-items.md index e7778a2f9c9..fa61d4e4875 100644 --- a/doc/release-notes/10943-featured-items.md +++ b/doc/release-notes/10943-featured-items.md @@ -7,4 +7,9 @@ CRUD endpoints for Collection Featured Items have been implemented. In particula - Delete all featured items in a collection (DELETE /api/dataverses//featuredItems) - Update all featured items in a collection (PUT /api/dataverses//featuredItems) +New settings: + +- dataverse.files.featured-items.image-maxsize - It sets the maximum allowed size of the image that can be added to a featured item. +- dataverse.files.featured-items.image-uploads - It specifies the name of the subdirectory for saving featured item images within the docroot directory. + See also #10943 and #11124. From e0aeb2f31bd89f0ea22298b4e64a9f512c5eeeed Mon Sep 17 00:00:00 2001 From: GPortas Date: Mon, 27 Jan 2025 15:34:19 +0000 Subject: [PATCH 131/137] Fixed: DataversesIT featured items imageFileUrl assertion --- .../java/edu/harvard/iq/dataverse/api/DataversesIT.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index b87f3b5f6ea..825465fcd9e 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -1731,7 +1731,7 @@ public void testListFeaturedItems() { .body("data[1].displayOrder", equalTo(1)) .body("data[2].content", equalTo("Content 1")) .body("data[2].imageFileName", equalTo("coffeeshop.png")) - .body("data[2].imageFileUrl", containsString(UtilIT.getRestAssuredBaseUri() + "/api/access/dataverseFeatureItemImage/")) + .body("data[2].imageFileUrl", containsString("/api/access/dataverseFeaturedItemImage/")) .body("data[2].displayOrder", equalTo(2)) .statusCode(OK.getStatusCode()); @@ -1766,7 +1766,7 @@ public void testUpdateFeaturedItems() { .body("data.size()", equalTo(3)) .body("data[0].content", equalTo("Content 1")) .body("data[0].imageFileName", equalTo("coffeeshop.png")) - .body("data[0].imageFileUrl", containsString(baseUri + "/api/access/dataverseFeatureItemImage/")) + .body("data[0].imageFileUrl", containsString("/api/access/dataverseFeaturedItemImage/")) .body("data[0].displayOrder", equalTo(0)) .body("data[1].content", equalTo("Content 2")) .body("data[1].imageFileName", equalTo(null)) @@ -1799,7 +1799,7 @@ public void testUpdateFeaturedItems() { .body("data[0].displayOrder", equalTo(0)) .body("data[1].content", equalTo("Content 1 updated")) .body("data[1].imageFileName", equalTo("coffeeshop.png")) - .body("data[1].imageFileUrl", containsString(baseUri + "/api/access/dataverseFeatureItemImage/")) + .body("data[1].imageFileUrl", containsString("/api/access/dataverseFeaturedItemImage/")) .body("data[1].displayOrder", equalTo(1)) .body("data[2].content", equalTo("Content 3")) .body("data[2].imageFileName", equalTo(null)) @@ -1828,7 +1828,7 @@ public void testUpdateFeaturedItems() { .body("data.size()", equalTo(3)) .body("data[0].content", equalTo("Content 2")) .body("data[0].imageFileName", equalTo("coffeeshop.png")) - .body("data[0].imageFileUrl", containsString(baseUri + "/api/access/dataverseFeatureItemImage/")) + .body("data[0].imageFileUrl", containsString("/api/access/dataverseFeaturedItemImage/")) .body("data[0].displayOrder", equalTo(0)) .body("data[1].content", equalTo("Content 1 updated")) .body("data[1].imageFileName", equalTo(null)) From d3726f89781868441aef82b4a16ed3de587c1aa9 Mon Sep 17 00:00:00 2001 From: GPortas Date: Tue, 28 Jan 2025 08:31:00 +0000 Subject: [PATCH 132/137] Fixed: doc hierarchy --- doc/sphinx-guides/source/api/native-api.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 249da446c8c..6f2b425322a 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1286,9 +1286,6 @@ The following example creates one featured item and updates a second one, keepin -F "keepFile=false" -F "keepFile=true" \ "https://demo.dataverse.org/api/dataverses/root/featuredItems" -Dataverse Collection Featured Items ------------------------------------ - Update A Collection Featured Item ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 29944a0e2c22185c92abc44992e5cba7612401ba Mon Sep 17 00:00:00 2001 From: GPortas Date: Tue, 28 Jan 2025 08:34:13 +0000 Subject: [PATCH 133/137] Changed: featured items docs hierarchy --- doc/sphinx-guides/source/api/native-api.rst | 90 ++++++++++----------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 6f2b425322a..c2714b6ac40 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1156,8 +1156,8 @@ Use the ``/settings`` API to enable or disable the enforcement of storage quotas curl -X PUT -d 'true' http://localhost:8080/api/admin/settings/:UseStorageQuotas -List Collection Featured Items -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +List All Collection Featured Items +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ List the featured items configured for a given Dataverse collection ``id``: @@ -1175,49 +1175,6 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/dataverses/root/featuredItems" -Delete All Collection Featured Items -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Deletes the featured items configured for a given Dataverse collection ``id``: - -.. code-block:: bash - - export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - export SERVER_URL=https://demo.dataverse.org - export ID=root - - curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE "$SERVER_URL/api/dataverses/$ID/featuredItems" - -The fully expanded example above (without environment variables) looks like this: - -.. code-block:: bash - - curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/dataverses/root/featuredItems" - -Create a Collection Featured Item -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Creates a featured item in the given Dataverse collection ``id``: - -.. code-block:: bash - - export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - export IMAGE_FILENAME='image.png' - export CONTENT='Content for featured item.' - export DISPLAY_ORDER=1 - export SERVER_URL=https://demo.dataverse.org - export ID=root - - curl -H "X-Dataverse-key:$API_TOKEN" -X POST -F "file=@$IMAGE_FILENAME" -F "content=$CONTENT" -F "displayOrder=$DISPLAY_ORDER" "$SERVER_URL/api/dataverses/$ID/featuredItems" - -The fully expanded example above (without environment variables) looks like this: - -.. code-block:: bash - - curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST -F "file=@image.png" -F "content=Content for featured item." -F "displayOrder=1" "https://demo.dataverse.org/api/dataverses/root/featuredItems" - -A featured item may or may not contain an image. If you wish to create it without an image, omit the file parameter in the request. - Update All Collection Featured Items ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1286,6 +1243,49 @@ The following example creates one featured item and updates a second one, keepin -F "keepFile=false" -F "keepFile=true" \ "https://demo.dataverse.org/api/dataverses/root/featuredItems" +Delete All Collection Featured Items +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Deletes the featured items configured for a given Dataverse collection ``id``: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=root + + curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE "$SERVER_URL/api/dataverses/$ID/featuredItems" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/dataverses/root/featuredItems" + +Create A Collection Featured Item +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Creates a featured item in the given Dataverse collection ``id``: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export IMAGE_FILENAME='image.png' + export CONTENT='Content for featured item.' + export DISPLAY_ORDER=1 + export SERVER_URL=https://demo.dataverse.org + export ID=root + + curl -H "X-Dataverse-key:$API_TOKEN" -X POST -F "file=@$IMAGE_FILENAME" -F "content=$CONTENT" -F "displayOrder=$DISPLAY_ORDER" "$SERVER_URL/api/dataverses/$ID/featuredItems" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST -F "file=@image.png" -F "content=Content for featured item." -F "displayOrder=1" "https://demo.dataverse.org/api/dataverses/root/featuredItems" + +A featured item may or may not contain an image. If you wish to create it without an image, omit the file parameter in the request. + Update A Collection Featured Item ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 497cc5d97458f25ef3abeb9a56c533d395fe0d17 Mon Sep 17 00:00:00 2001 From: Guillermo Portas Date: Tue, 28 Jan 2025 13:43:28 +0000 Subject: [PATCH 134/137] Update doc/sphinx-guides/source/api/native-api.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index c2714b6ac40..ca389d366f9 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1262,7 +1262,7 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/dataverses/root/featuredItems" -Create A Collection Featured Item +Create a Collection Featured Item ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Creates a featured item in the given Dataverse collection ``id``: From 1799d70949bf6410c9672942bd04390e70025090 Mon Sep 17 00:00:00 2001 From: Guillermo Portas Date: Tue, 28 Jan 2025 13:43:35 +0000 Subject: [PATCH 135/137] Update doc/sphinx-guides/source/api/native-api.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index ca389d366f9..10c42e1b76c 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1286,7 +1286,7 @@ The fully expanded example above (without environment variables) looks like this A featured item may or may not contain an image. If you wish to create it without an image, omit the file parameter in the request. -Update A Collection Featured Item +Update a Collection Featured Item ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Updates a featured item given its ``id``: From 1aac2d511c8f37f61aaa8930f77540434bc8bac9 Mon Sep 17 00:00:00 2001 From: Guillermo Portas Date: Tue, 28 Jan 2025 13:43:41 +0000 Subject: [PATCH 136/137] Update doc/sphinx-guides/source/api/native-api.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 10c42e1b76c..212eec2603e 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1322,7 +1322,7 @@ Updating the featured item removing the existing image: curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT -F "content=Content for featured item." -F "displayOrder=1" "https://demo.dataverse.org/api/dataverseFeaturedItems/1" -Delete A Collection Featured Item +Delete a Collection Featured Item ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Deletes a featured item given its ``id``: From a8d4a0526054567b150646195fed58d51a6b1e14 Mon Sep 17 00:00:00 2001 From: Guillermo Portas Date: Tue, 28 Jan 2025 13:43:48 +0000 Subject: [PATCH 137/137] Update doc/sphinx-guides/source/api/native-api.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 212eec2603e..76682d1cec8 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1341,7 +1341,7 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/dataverseFeaturedItems/1" -Get A Collection Featured Item Image +Get a Collection Featured Item Image ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Returns the image of a featured item if one is assigned, given the featured item ``id``: