Commit a84ee81d authored by Matija Obreza's avatar Matija Obreza
Browse files

Store ISO639 vocabulary

- POST to /api/v0/lang/update to update the vocabulary
parent 1c2cdf1d
......@@ -21,6 +21,7 @@ import org.genesys.catalog.model.vocab.ControlledVocabulary;
import org.genesys.catalog.service.filters.ControlledVocabularyFilter;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.security.access.prepost.PreAuthorize;
/**
* The VocabularyService.
......@@ -58,6 +59,7 @@ public interface VocabularyService {
* @param input updated vocabulary definition
* @return persisted vocabulary
*/
@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#input, 'write')")
ControlledVocabulary updateVocabulary(ControlledVocabulary input);
/**
......@@ -66,6 +68,7 @@ public interface VocabularyService {
* @param vocabulary the vocabulary
* @return the controlled vocabulary
*/
@PreAuthorize("hasRole('ADMINISTRATOR') or hasPermission(#input, 'delete')")
ControlledVocabulary deleteVocabulary(ControlledVocabulary vocabulary);
/**
......
......@@ -39,7 +39,7 @@ import org.springframework.stereotype.Component;
*/
@Component
public class GeonamesISOLanguageSource {
private static final String GEONAMES_ISO639_URL = "http://download.geonames.org/export/dump/iso-languagecodes.txt";
public static final String GEONAMES_ISO639_URL = "http://download.geonames.org/export/dump/iso-languagecodes.txt";
/** The Constant LOG. */
public static final Logger LOG = LoggerFactory.getLogger(GeonamesISOLanguageSource.class);
......
......@@ -21,6 +21,7 @@ import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
import java.util.function.Function;
import org.apache.commons.logging.Log;
......@@ -39,56 +40,63 @@ import org.springframework.stereotype.Component;
@Component
public class ISO639VocabularyUpdater {
/** The Constant LOG. */
public static final Log LOG = LogFactory.getLog(ISO639VocabularyUpdater.class);
/**
* ISO 639 representation of names for languages and language groups.
* https://en.wikipedia.org/wiki/ISO_639
*/
public static UUID ISO639_3 = UUID.fromString("21b10067-ba15-44dd-867f-6a18a117fee8");
@Autowired
private GeonamesISOLanguageSource isoLanguageSource;
/** The Constant LOG. */
public static final Log LOG = LogFactory.getLog(ISO639VocabularyUpdater.class);
/**
* Generates a current ISO639-3 {@link ControlledVocabulary} but doesn't
* persist it to storage.
*
* @return vocabulary of ISO639-3 3-letter language codes
* @throws IOException IOException
*/
public ControlledVocabulary getISO639Vocabulary() throws IOException {
return createVocabulary("ISO639-3", LanguageInfo::getCode);
}
@Autowired
private GeonamesISOLanguageSource isoLanguageSource;
/**
* Creates the vocabulary.
*
* @param title the title
* @param toTerm the to term
* @return the controlled vocabulary
* @throws IOException Signals that an I/O exception has occurred.
*/
protected ControlledVocabulary createVocabulary(final String title, final Function<LanguageInfo, String> toTerm) throws IOException {
final ControlledVocabulary vocabulary = new ControlledVocabulary();
vocabulary.setTitle(title);
/**
* Generates a current ISO639-3 {@link ControlledVocabulary} but doesn't persist
* it to storage.
*
* @return vocabulary of ISO639-3 3-letter language codes
* @throws IOException IOException
*/
public ControlledVocabulary getISO639Vocabulary() throws IOException {
return createVocabulary("ISO639-3", LanguageInfo::getCode);
}
final DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyy.MM.dd");
final LocalDate localDate = LocalDate.now();
vocabulary.setVersionTag(dtf.format(localDate));
/**
* Creates the vocabulary.
*
* @param title the title
* @param toTerm the to term
* @return the controlled vocabulary
* @throws IOException Signals that an I/O exception has occurred.
*/
protected ControlledVocabulary createVocabulary(final String title, final Function<LanguageInfo, String> toTerm) throws IOException {
final ControlledVocabulary vocabulary = new ControlledVocabulary();
vocabulary.setTitle(title);
vocabulary.setUrl(GeonamesISOLanguageSource.GEONAMES_ISO639_URL);
final Map<String, VocabularyTerm> assignedCodes = new HashMap<>();
isoLanguageSource.fetchLanguageData().stream().map(language -> {
final VocabularyTerm term = new VocabularyTerm();
term.setCode(toTerm.apply(language));
term.setTitle(language.getLanguage());
return term;
})
// remove terms without codes
.filter(term -> (term.getCode() != null) && (term.getCode().length() > 0))
// add to vocabularyTerms if ISO language code is not assigned
.forEach(term -> {
if (!assignedCodes.containsKey(term.getCode())) {
assignedCodes.put(term.getCode(), term);
}
});
final DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyy.MM.dd");
final LocalDate localDate = LocalDate.now();
vocabulary.setVersionTag(dtf.format(localDate));
vocabulary.setTerms(new ArrayList<>(assignedCodes.values()));
return vocabulary;
}
final Map<String, VocabularyTerm> assignedCodes = new HashMap<>();
isoLanguageSource.fetchLanguageData().stream().map(language -> {
final VocabularyTerm term = new VocabularyTerm();
term.setCode(toTerm.apply(language));
term.setTitle(language.getLanguage());
return term;
})
// remove terms without codes
.filter(term -> (term.getCode() != null) && (term.getCode().length() > 0))
// add to vocabularyTerms if ISO language code is not assigned
.forEach(term -> {
if (!assignedCodes.containsKey(term.getCode())) {
assignedCodes.put(term.getCode(), term);
}
});
vocabulary.setTerms(new ArrayList<>(assignedCodes.values()));
return vocabulary;
}
}
......@@ -16,21 +16,26 @@
package org.genesys.catalog.server.controller.api.v0;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
import java.util.stream.Collectors;
import org.genesys.catalog.exceptions.NotFoundElement;
import org.genesys.catalog.model.vocab.ControlledVocabulary;
import org.genesys.catalog.model.vocab.VocabularyTerm;
import org.genesys.catalog.service.worker.GeonamesISOLanguageSource.LanguageInfo;
import org.genesys.catalog.service.PartnerService;
import org.genesys.catalog.service.VocabularyService;
import org.genesys.catalog.service.worker.ISO639VocabularyUpdater;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.MediaType;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
......@@ -43,33 +48,63 @@ import org.springframework.web.bind.annotation.RestController;
@RestController
@RequestMapping(LanguagesController.API_BASE)
@PreAuthorize("isAuthenticated()")
public class LanguagesController {
protected static final String API_BASE = "/api/v0/lang";
private static final Logger LOG = LoggerFactory.getLogger(LanguagesController.class);
@Autowired
private ISO639VocabularyUpdater iso639VocabularyUpdater;
@GetMapping(value = "/{code}", produces = MediaType.APPLICATION_JSON_VALUE)
public LanguageInfo get(@PathVariable("code") final String code) throws IOException {
final VocabularyTerm vocabularyTerm = iso639VocabularyUpdater.getISO639Vocabulary().getTerms().stream()
.filter(term -> term.getCode().toLowerCase().equals(code.toLowerCase())).findFirst()
.orElseThrow(() -> new NotFoundElement("Language with such code not found"));
return new LanguageInfo(vocabularyTerm.getCode(), vocabularyTerm.getTitle());
}
@GetMapping(value = "/autocomplete", produces = MediaType.APPLICATION_JSON_VALUE)
public List<LanguageInfo> autocomplete(@RequestParam("l") final String languageLike) throws IOException {
if (languageLike.length() < 3)
return Collections.emptyList();
final List<LanguageInfo> languages = new ArrayList<>(20);
iso639VocabularyUpdater.getISO639Vocabulary().getTerms().stream()
.filter(term -> term.getTitle().toLowerCase().startsWith(languageLike.toLowerCase()))
.limit(20)
.forEach(term -> languages.add(new LanguageInfo(term.getCode(), term.getTitle())));
return languages;
}
public class LanguagesController implements InitializingBean {
protected static final String API_BASE = "/api/v0/lang";
public static final UUID ISO639_3 = ISO639VocabularyUpdater.ISO639_3;
private static final Logger LOG = LoggerFactory.getLogger(LanguagesController.class);
@Autowired
private ISO639VocabularyUpdater iso639VocabularyUpdater;
@Autowired
private VocabularyService vocabularyService;
@Autowired
private PartnerService partnerService;
@Override
public void afterPropertiesSet() throws Exception {
final ControlledVocabulary iso639 = vocabularyService.getVocabulary(ISO639_3);
if (iso639 == null) {
ControlledVocabulary updated = iso639VocabularyUpdater.getISO639Vocabulary();
updated.setUuid(ISO639_3);
updated.setOwner(partnerService.getPrimaryPartner());
updated = vocabularyService.createVocabulary(updated);
LOG.info("Created ISO639-3 vocabulary with {} terms", updated.getTerms().size());
}
}
@PreAuthorize("hasRole('ADMINISTRATOR')")
@PostMapping(value = "/update")
public ControlledVocabulary updateLanguages() throws IOException {
final ControlledVocabulary iso639 = vocabularyService.getVocabulary(ISO639_3);
ControlledVocabulary updated = iso639VocabularyUpdater.getISO639Vocabulary();
updated.setUuid(iso639.getUuid());
updated.setVersion(iso639.getVersion());
LOG.info("Updating ISO639-3 vocabulary with {} terms", updated.getTerms().size());
return vocabularyService.updateVocabulary(updated);
}
@GetMapping(value = "/{code}", produces = MediaType.APPLICATION_JSON_VALUE)
public VocabularyTerm get(@PathVariable("code") final String code) {
ControlledVocabulary iso639 = vocabularyService.getVocabulary(ISO639_3);
return iso639.getTerms().stream().filter(term -> term.getCode().toLowerCase().equals(code.toLowerCase())).findFirst().orElseThrow(() -> new NotFoundElement(
"Language with such code not found"));
}
@GetMapping(value = "/autocomplete", produces = MediaType.APPLICATION_JSON_VALUE)
public List<VocabularyTerm> autocomplete(@RequestParam("l") final String languageLike) throws IOException {
if (languageLike.length() < 3)
return Collections.emptyList();
ControlledVocabulary iso639 = vocabularyService.getVocabulary(ISO639_3);
return iso639.getTerms().stream().filter(term -> term.getTitle().toLowerCase().startsWith(languageLike.toLowerCase())).limit(20).collect(Collectors.toList());
}
}
......@@ -15,27 +15,27 @@
*/
package org.genesys.catalog.server.controller.api.v0;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.nullValue;
import static org.springframework.restdocs.mockmvc.MockMvcRestDocumentation.documentationConfiguration;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import org.genesys.catalog.server.controller.rest.AbstractRestTest;
import org.genesys.catalog.server.controller.rest.WithMockOAuth2Authentication;
import org.genesys.catalog.service.worker.GeonamesISOLanguageSource.LanguageInfo;
import org.junit.Before;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.MediaType;
import org.springframework.restdocs.mockmvc.RestDocumentationRequestBuilders;
import org.springframework.security.test.context.support.WithMockUser;
import org.springframework.test.web.servlet.MockMvc;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.context.WebApplicationContext;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.nullValue;
import static org.springframework.restdocs.mockmvc.MockMvcRestDocumentation.documentationConfiguration;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
/**
* The Class LanguagesControllerTest.
*
......@@ -43,36 +43,36 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.
*/
public class LanguagesControllerTest extends AbstractRestTest {
@Autowired
private WebApplicationContext webApplicationContext;
@Autowired
private WebApplicationContext webApplicationContext;
private MockMvc mockMvc;
private MockMvc mockMvc;
@Before
@Transactional
public void setUp() {
mockMvc = MockMvcBuilders.webAppContextSetup(webApplicationContext).apply(documentationConfiguration(restDocumentation).uris().withScheme("https").withHost(
"api.catalog.genesys-pgr.org").withPort(443)).build();
}
@Before
@Transactional
public void setUp() throws Exception {
mockMvc = MockMvcBuilders.webAppContextSetup(webApplicationContext).apply(documentationConfiguration(restDocumentation).uris().withScheme("https").withHost(
"api.catalog.genesys-pgr.org").withPort(443)).build();
}
@Test
@WithMockOAuth2Authentication()
public void getLanguageInfoByCodeTest() throws Exception {
final LanguageInfo languageInfo = new LanguageInfo("ukr", "Ukrainian");
/*@formatter:off*/
@Test
@WithMockUser(username = "user", password = "user", roles = "ADMINISTRATOR")
public void getLanguageInfoByCodeTest() throws Exception {
final LanguageInfo languageInfo = new LanguageInfo("ukr", "Ukrainian");
/*@formatter:off*/
mockMvc
.perform(RestDocumentationRequestBuilders.get(LanguagesController.API_BASE.concat("/{code}"), languageInfo.getCode()))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
.andExpect(jsonPath("$.code", is(languageInfo.getCode())))
.andExpect(jsonPath("$.language", is(languageInfo.getLanguage())));
.andExpect(jsonPath("$.title", is(languageInfo.getLanguage())));
/*@formatter:on*/
}
}
@Test
@WithMockOAuth2Authentication()
public void listLanguageInfoTest() throws Exception {
final LanguageInfo languageInfo = new LanguageInfo("ukr", "Ukrainian");
@Test
@WithMockUser(username = "user", password = "user", roles = "ADMINISTRATOR")
public void listLanguageInfoTest() throws Exception {
final LanguageInfo languageInfo = new LanguageInfo("ukr", "Ukrainian");
/*@formatter:off*/
mockMvc
.perform(RestDocumentationRequestBuilders.get(LanguagesController.API_BASE.concat("/autocomplete"))
......@@ -82,14 +82,14 @@ public class LanguagesControllerTest extends AbstractRestTest {
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
.andExpect(jsonPath("$.[0]", not(nullValue())))
.andExpect(jsonPath("$.[0].code", not(nullValue())))
.andExpect(jsonPath("$.[0].language", not(nullValue())));
.andExpect(jsonPath("$.[0].title", not(nullValue())));
/*@formatter:on*/
}
}
@Test
@WithMockOAuth2Authentication()
public void searchTermToShortTest() throws Exception {
final LanguageInfo languageInfo = new LanguageInfo("ukr", "Ukrainian");
@Test
@WithMockUser(username = "user", password = "user", roles = "ADMINISTRATOR")
public void searchTermToShortTest() throws Exception {
final LanguageInfo languageInfo = new LanguageInfo("ukr", "Ukrainian");
/*@formatter:off*/
mockMvc
.perform(RestDocumentationRequestBuilders.get(LanguagesController.API_BASE.concat("/autocomplete"))
......@@ -99,5 +99,5 @@ public class LanguagesControllerTest extends AbstractRestTest {
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
.andExpect(content().string("[]"));
/*@formatter:on*/
}
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment