From 11c7f1ee5b26d05a1e3adf625d5312bdab598ddf Mon Sep 17 00:00:00 2001 From: christophe mangeat Date: Mon, 29 Jan 2024 16:10:53 +0100 Subject: [PATCH 37/43] [gn] on subtemplate's update, trigger indexation for every md referencing it rely on metadataIndexer.batchIndexInThreadPool for async indexing test threadPoolIndexation starts for expected md throw exception when updating a subtemplate too resource hungry (too many md to index, i.e. more than 10000) --- .../fao/geonet/exceptions/GNException.java | 34 ++++++ .../datamanager/base/BaseMetadataManager.java | 35 +++++- .../kernel/LocalXLinksUpdateDeleteTest.java | 107 ++++++++++++++---- .../mocked-core-repository-test-context.xml | 5 + 4 files changed, 156 insertions(+), 25 deletions(-) create mode 100644 common/src/main/java/org/fao/geonet/exceptions/GNException.java diff --git a/common/src/main/java/org/fao/geonet/exceptions/GNException.java b/common/src/main/java/org/fao/geonet/exceptions/GNException.java new file mode 100644 index 00000000000..cedd2239e43 --- /dev/null +++ b/common/src/main/java/org/fao/geonet/exceptions/GNException.java @@ -0,0 +1,34 @@ +//============================================================================= +//=== Copyright (C) 2001-2023 Food and Agriculture Organization of the +//=== United Nations (FAO-UN), United Nations World Food Programme (WFP) +//=== and United Nations Environment Programme (UNEP) +//=== +//=== This program is free software; you can redistribute it and/or modify +//=== it under the terms of the GNU General Public License as published by +//=== the Free Software Foundation; either version 2 of the License, or (at +//=== your option) any later version. +//=== +//=== This program is distributed in the hope that it will be useful, but +//=== WITHOUT ANY WARRANTY; without even the implied warranty of +//=== MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +//=== General Public License for more details. +//=== +//=== You should have received a copy of the GNU General Public License +//=== along with this program; if not, write to the Free Software +//=== Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA +//=== +//=== Contact: Jeroen Ticheler - FAO - Viale delle Terme di Caracalla 2, +//=== Rome - Italy. email: geonetwork@osgeo.org +//============================================================================== +package org.fao.geonet.exceptions; + +public class GNException extends RuntimeException { + + public GNException(String msg) { + super(msg); + } + + public GNException(String msg, Throwable cause) { + super(msg, cause); + } +} diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java index ba128b9c033..ac6165935c1 100644 --- a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java @@ -23,6 +23,7 @@ package org.fao.geonet.kernel.datamanager.base; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Function; import com.google.common.base.Optional; import com.google.common.base.Predicate; @@ -33,11 +34,13 @@ import jeeves.transaction.TransactionTask; import jeeves.xlink.Processor; import org.apache.commons.lang.StringUtils; +import org.elasticsearch.action.search.SearchResponse; import org.fao.geonet.ApplicationContextHolder; import org.fao.geonet.constants.Edit; import org.fao.geonet.constants.Geonet; import org.fao.geonet.constants.Params; import org.fao.geonet.domain.*; +import org.fao.geonet.exceptions.GNException; import org.fao.geonet.exceptions.UnAuthorizedException; import org.fao.geonet.kernel.*; import org.fao.geonet.kernel.datamanager.*; @@ -45,7 +48,6 @@ import org.fao.geonet.kernel.schema.SchemaPlugin; import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; -import org.fao.geonet.kernel.search.MetaSearcher; import org.fao.geonet.kernel.search.index.BatchOpsMetadataReindexer; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; @@ -104,8 +106,6 @@ public class BaseMetadataManager implements IMetadataManager { @Autowired private GroupRepository groupRepository; @Autowired - private MetadataStatusRepository metadataStatusRepository; - @Autowired private MetadataValidationRepository metadataValidationRepository; @Autowired private MetadataRepository metadataRepository; @@ -141,6 +141,10 @@ public class BaseMetadataManager implements IMetadataManager { @Autowired private ApplicationContext _applicationContext; + + @VisibleForTesting + public int maxMdsReferencingSubTemplate = 10000; + @PersistenceContext private EntityManager _entityManager; @@ -747,6 +751,9 @@ public synchronized AbstractMetadata updateMetadata(final ServiceContext context getSearchManager().delete(String.format("+uuid:\"%s\"", uuidBeforeUfo)); } metadataIndexer.indexMetadata(metadataId, true, indexingMode); + if (metadata.getDataInfo().getType() == MetadataType.SUB_TEMPLATE) { + indexMdsReferencingSubTemplate(context, metadata); + } } } @@ -1314,4 +1321,26 @@ boolean hasReferencingMetadata(ServiceContext context, AbstractMetadata metadata return this.searchManager.query(query.toString(), null, 0, 0).getHits().getTotalHits().value > 0; } + private void indexMdsReferencingSubTemplate(ServiceContext context, AbstractMetadata subTemplate) throws Exception { + StringBuilder query = new StringBuilder(String.format("xlink:*%s*", subTemplate.getUuid())); + SearchResponse response = this.searchManager.query(query.toString(), null, 0, maxMdsReferencingSubTemplate); + if (response.getHits().getTotalHits().value > maxMdsReferencingSubTemplate) { + throw new GNException("Not implemented"); + } + ArrayList toIndex = new ArrayList<>(); + response.getHits().forEach(consumer -> { + String consumerUuid = consumer.getId(); + try { + String consumerId = this.metadataUtils.getMetadataId(consumerUuid); + if (consumerId != null) { + toIndex.add(consumerId); + } + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + metadataIndexer.batchIndexInThreadPool(context, toIndex); + } } diff --git a/core/src/test/java/org/fao/geonet/kernel/LocalXLinksUpdateDeleteTest.java b/core/src/test/java/org/fao/geonet/kernel/LocalXLinksUpdateDeleteTest.java index e67834f85ef..cf63907de4f 100644 --- a/core/src/test/java/org/fao/geonet/kernel/LocalXLinksUpdateDeleteTest.java +++ b/core/src/test/java/org/fao/geonet/kernel/LocalXLinksUpdateDeleteTest.java @@ -1,9 +1,15 @@ package org.fao.geonet.kernel; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import jeeves.server.context.ServiceContext; +import org.elasticsearch.action.search.SearchResponse; import org.fao.geonet.AbstractCoreIntegrationTest; import org.fao.geonet.domain.AbstractMetadata; -import org.fao.geonet.kernel.datamanager.IMetadataManager; +import org.fao.geonet.exceptions.GNException; +import org.fao.geonet.kernel.datamanager.base.BaseMetadataIndexer; +import org.fao.geonet.kernel.datamanager.base.BaseMetadataManager; +import org.fao.geonet.kernel.search.EsSearchManager; import org.fao.geonet.kernel.search.IndexingMode; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; @@ -12,30 +18,43 @@ import org.jdom.Element; import org.junit.Before; import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import java.net.URL; import java.util.Arrays; +import java.util.Collections; +import java.util.List; import static org.fao.geonet.domain.MetadataType.SUB_TEMPLATE; import static org.fao.geonet.domain.MetadataType.TEMPLATE; import static org.fao.geonet.schema.iso19139.ISO19139Namespaces.GCO; import static org.fao.geonet.schema.iso19139.ISO19139Namespaces.GMD; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.when; - public class LocalXLinksUpdateDeleteTest extends AbstractIntegrationTestWithMockedSingletons { @Autowired - private IMetadataManager metadataManager; + private BaseMetadataManager metadataManager; @Autowired private SettingManager settingManager; + @Autowired + private EsSearchManager searchManager; + private ServiceContext serviceContext; + @Autowired + private BaseMetadataIndexer metadataIndexer; + @Before public void setUp() throws Exception { serviceContext = createServiceContext(); @@ -44,28 +63,42 @@ public void setUp() throws Exception { @Test public void updateHasToRegisterReferrersForIndexation() throws Exception { - URL contactResource = AbstractCoreIntegrationTest.class.getResource("kernel/babarContact.xml"); - Element contactElement = Xml.loadStream(contactResource.openStream()); - AbstractMetadata contactMetadata = insertContact(contactElement); - AbstractMetadata vicinityMapMetadata = insertVicinityMap(contactMetadata); + Mockito.reset(metadataIndexer); - // TODOES -// Object document = searchForMetadataTagged("babar"); -// assertFalse(context.getBean(IndexingList.class).getIdentifiers().contains(vicinityMapMetadata.getId())); + AbstractMetadata vicinityMapMetadata = updateASubtemplateUsedByOneMetadata(); - Xml.selectElement(contactElement, "gmd:individualName/gco:CharacterString", Arrays.asList(GMD, GCO)).setText("momo"); - metadataManager.updateMetadata(serviceContext, - Integer.toString(contactMetadata.getId()), - contactElement, - false, - false, - null, - null, - false, - IndexingMode.full); + ArgumentCaptor toIndexCaptor = ArgumentCaptor.forClass(List.class); + Mockito.verify(metadataIndexer).batchIndexInThreadPool(Mockito.anyObject(), toIndexCaptor.capture()); + assertTrue(toIndexCaptor.getValue().contains(Integer.toString(vicinityMapMetadata.getId()))); + } + + @Test + public void exceedMaxMdsReferencingSubTemplateThrowsException() { + Mockito.reset(metadataIndexer); + try { + metadataManager.maxMdsReferencingSubTemplate = 0; + + assertThrows(GNException.class, () -> updateASubtemplateUsedByOneMetadata()); + + Mockito.verify(metadataIndexer, never()).batchIndexInThreadPool(Mockito.anyObject(), Mockito.anyList()); + } finally { + metadataManager.maxMdsReferencingSubTemplate = 10000; + } + } + + private SearchResponse countMetadataUsing(String mdUuid, String contactForResourceIndividual) throws Exception { + ObjectMapper objectMapper = new ObjectMapper(); + + JsonNode query = objectMapper.createObjectNode() + .set("nested", objectMapper.createObjectNode() + .put("path", "contactForResource") + .set("query", objectMapper.createObjectNode() + .set("bool", objectMapper.createObjectNode().set("must", objectMapper.createArrayNode() + .add(objectMapper.createObjectNode().set("match", objectMapper.createObjectNode().put("_id", mdUuid))) + .add(objectMapper.createObjectNode().set("match", objectMapper.createObjectNode().put("contactForResource.individual", contactForResourceIndividual)))))) + ); -// TODOES assertEquals(vicinityMapMetadata.getUuid(), document.getField("_uuid").stringValue()); -// assertTrue(context.getBean(IndexingList.class).getIdentifiers().contains(vicinityMapMetadata.getId())); + return this.searchManager.query(query, Collections.emptySet(), 0, 10, null); } @Test @@ -124,4 +157,34 @@ private AbstractMetadata insertContact(Element contactElement) throws Exception when(springLocalServiceInvoker.invoke(any(String.class))).thenReturn(contactElement); return contactMetadata; } + + private AbstractMetadata updateASubtemplateUsedByOneMetadata() throws Exception { + URL contactResource = AbstractCoreIntegrationTest.class.getResource("kernel/babarContact.xml"); + Element contactElement = Xml.loadStream(contactResource.openStream()); + AbstractMetadata contactMetadata = insertContact(contactElement); + AbstractMetadata vicinityMapMetadata = insertVicinityMap(contactMetadata); + assertEquals(1, countMetadataUsing(vicinityMapMetadata.getUuid(), "babar").getHits().getHits().length); + assertEquals(0, countMetadataUsing(vicinityMapMetadata.getUuid(), "momo").getHits().getHits().length); + + Xml.selectElement(contactElement, "gmd:individualName/gco:CharacterString", Arrays.asList(GMD, GCO)).setText("momo"); + metadataManager.updateMetadata(serviceContext, + Integer.toString(contactMetadata.getId()), + contactElement, + false, + false, + null, + null, + false, + IndexingMode.full); + return vicinityMapMetadata; + } + + static public class SpyFactory { + @Autowired + BaseMetadataIndexer baseMetadataIndexer; + + BaseMetadataIndexer build() { + return Mockito.spy(baseMetadataIndexer); + } + } } diff --git a/core/src/test/resources/mocked-core-repository-test-context.xml b/core/src/test/resources/mocked-core-repository-test-context.xml index b3d835c52ad..9d723700b9a 100644 --- a/core/src/test/resources/mocked-core-repository-test-context.xml +++ b/core/src/test/resources/mocked-core-repository-test-context.xml @@ -30,4 +30,9 @@ + + + + + From ae6fddfd1d3323612eafe4148bac6ea80e48dee7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cbenoitregamey=E2=80=9D?= <“benoitregamey@hotmail.com”> Date: Tue, 16 Jan 2024 12:10:34 +0100 Subject: [PATCH 38/43] [geocat] change basemap to greyscale swisstopo in EPSG:3857 --- .../data/data/resources/map/config-viewer.xml | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/web/src/main/webapp/WEB-INF/data/data/resources/map/config-viewer.xml b/web/src/main/webapp/WEB-INF/data/data/resources/map/config-viewer.xml index a2a43f0eca9..c47393ac185 100644 --- a/web/src/main/webapp/WEB-INF/data/data/resources/map/config-viewer.xml +++ b/web/src/main/webapp/WEB-INF/data/data/resources/map/config-viewer.xml @@ -2,19 +2,30 @@ xmlns:ows="http://www.opengis.net/ows" version="0.3.1" id="ows-context-ex-1-v3"> - - 426340.7642533947 -31175.881660288083 - 897081.2598204871 439564.6139068043 + + 500000 5600000 + 1390000 6150000 - + + Date: Tue, 16 Jan 2024 12:11:45 +0100 Subject: [PATCH 39/43] [geocat] set global CRS to EPSG:3857 and add support for EPSG:2056 (LV95) --- .../resources/catalog/js/CatController.js | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/web-ui/src/main/resources/catalog/js/CatController.js b/web-ui/src/main/resources/catalog/js/CatController.js index 0bf09eab66f..0acaae56291 100644 --- a/web-ui/src/main/resources/catalog/js/CatController.js +++ b/web-ui/src/main/resources/catalog/js/CatController.js @@ -841,7 +841,7 @@ wms: [], wmts: [] }, - projection: "EPSG:21781", + projection: "EPSG:3857", projectionList: [ { code: "urn:ogc:def:crs:EPSG:6.6:4326", @@ -853,16 +853,37 @@ }, { code: "EPSG:21781", - label: "Swiss LV03(EPSG:21781)" + label: "Swiss LV03 (EPSG:21781)" + }, + { + code: "EPSG:2056", + label: "Swiss LV95 (EPSG:2056)" } ], switcherProjectionList: [ { code: "EPSG:3857", - label: "Google mercator (EPSG:3857)", + label: "Google mercator (EPSG:3857)" + }, + { code: "EPSG:21781", - def: "+proj=somerc +lat_0=46.95240555555556 +lon_0=7.439583333333333 +k_0=1 +x_0=600000 +y_0=200000 +ellps=bessel +towgs84=674.4,15.1,405.3,0,0,0,0 +units=m +no_defs", - label: "Swiss LV03(EPSG:21781)" + label: "CH1903 / LV03", + def: "+proj=somerc +lat_0=46.9524055555556 +lon_0=7.43958333333333 +k_0=1 +x_0=600000 +y_0=200000 +ellps=bessel +towgs84=674.374,15.056,405.346,0,0,0,0 +units=m +no_defs +type=crs", + extent: [ + 485071.57525302575, 74284.81278939791, 837119.8047146694, + 299941.78643935296 + ], + worldExtent: [5.96, 45.82, 10.49, 47.81] + }, + { + label: "CH1903+ / LV95", + code: "EPSG:2056", + extent: [ + 2485071.5752530256, 1074284.812789398, 2837119.8047146695, + 1299941.786439353 + ], + worldExtent: [5.96, 45.82, 10.49, 47.81], + def: "+proj=somerc +lat_0=46.9524055555556 +lon_0=7.43958333333333 +k_0=1 +x_0=2600000 +y_0=1200000 +ellps=bessel +towgs84=674.374,15.056,405.346,0,0,0,0 +units=m +no_defs +type=crs" } ], disabledTools: { From 8c1ef640911e8ab987144710709127b069af83c3 Mon Sep 17 00:00:00 2001 From: Olivia Date: Thu, 20 Jun 2024 14:32:20 +0200 Subject: [PATCH 40/43] [geocat] change logo to newer version, hide link to new version in login screen --- .../resources/catalog/style/gn_search.less | 7 ++++++- .../templates/top-toolbar-accessible.html | 2 +- .../catalog/views/default/images/cat-logo.png | Bin 17559 -> 0 bytes .../views/default/images/geocat_logo.png | Bin 31945 -> 21326 bytes .../catalog/views/default/templates/home.html | 7 +------ 5 files changed, 8 insertions(+), 8 deletions(-) delete mode 100644 web-ui/src/main/resources/catalog/views/default/images/cat-logo.png diff --git a/web-ui/src/main/resources/catalog/style/gn_search.less b/web-ui/src/main/resources/catalog/style/gn_search.less index 0d5efc5ac86..b65ac32a35b 100644 --- a/web-ui/src/main/resources/catalog/style/gn_search.less +++ b/web-ui/src/main/resources/catalog/style/gn_search.less @@ -19,9 +19,10 @@ margin-right: -60px; } .gcat-logo { + height: 100px; display: flex; flex-direction: row; - justify-content: space-between; + justify-content: center; padding: 0px 16px; & > div { flex-shrink: 1; @@ -31,6 +32,10 @@ max-width: 100%; } } + + @media (min-width: @screen-sm-max) { + height: 168px; + } } } //GEOCAT diff --git a/web-ui/src/main/resources/catalog/templates/top-toolbar-accessible.html b/web-ui/src/main/resources/catalog/templates/top-toolbar-accessible.html index 082f587b219..d9a60745c49 100644 --- a/web-ui/src/main/resources/catalog/templates/top-toolbar-accessible.html +++ b/web-ui/src/main/resources/catalog/templates/top-toolbar-accessible.html @@ -203,7 +203,7 @@