Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Genesys PGR
Genesys Backend
Commits
0abdbf0c
Commit
0abdbf0c
authored
Dec 18, 2014
by
Matija Obreza
Browse files
Load data in batches for MCPD export
parent
c054dac5
Changes
11
Hide whitespace changes
Inline
Side-by-side
src/main/java/org/genesys2/server/persistence/domain/AccessionAliasRepository.java
View file @
0abdbf0c
...
...
@@ -38,4 +38,7 @@ public interface AccessionAliasRepository extends JpaRepository<AccessionAlias,
@Query
(
"select distinct aa.accession from AccessionAlias aa where aa.usedBy=?1 and aa.name=?2 and aa.aliasType=?3 and aa.accession.institute.code=?1"
)
Accession
findAccession
(
String
instCode
,
String
name
,
int
aliasType
);
@Query
(
"from AccessionAlias aa where aa.accession.id in ?1"
)
List
<
AccessionAlias
>
findAllFor
(
List
<
Long
>
accessionIds
);
}
src/main/java/org/genesys2/server/persistence/domain/AccessionBreedingRepository.java
View file @
0abdbf0c
...
...
@@ -17,6 +17,7 @@
package
org.genesys2.server.persistence.domain
;
import
java.util.Collection
;
import
java.util.List
;
import
org.genesys2.server.model.genesys.Accession
;
import
org.genesys2.server.model.genesys.AccessionBreeding
;
...
...
@@ -31,4 +32,7 @@ public interface AccessionBreedingRepository extends JpaRepository<AccessionBree
@Modifying
@Query
(
"delete from AccessionBreeding ab where ab.accession.id in ?1"
)
void
deleteForAccessions
(
Collection
<
Long
>
accessionIds
);
@Query
(
"from AccessionBreeding ab where ab.accession.id in ?1"
)
List
<
AccessionBreeding
>
findAllFor
(
List
<
Long
>
accessionIds
);
}
src/main/java/org/genesys2/server/persistence/domain/AccessionCollectRepository.java
View file @
0abdbf0c
...
...
@@ -17,6 +17,7 @@
package
org.genesys2.server.persistence.domain
;
import
java.util.Collection
;
import
java.util.List
;
import
org.genesys2.server.model.genesys.Accession
;
import
org.genesys2.server.model.genesys.AccessionCollect
;
...
...
@@ -31,4 +32,7 @@ public interface AccessionCollectRepository extends JpaRepository<AccessionColle
@Modifying
@Query
(
"delete from AccessionCollect ac where ac.accession.id in ?1"
)
void
deleteForAccessions
(
Collection
<
Long
>
accessionIds
);
@Query
(
"from AccessionCollect ac where ac.accession.id in ?1"
)
List
<
AccessionCollect
>
findAllFor
(
List
<
Long
>
accessionId
);
}
src/main/java/org/genesys2/server/persistence/domain/AccessionExchangeRepository.java
View file @
0abdbf0c
...
...
@@ -17,6 +17,7 @@
package
org.genesys2.server.persistence.domain
;
import
java.util.Collection
;
import
java.util.List
;
import
org.genesys2.server.model.genesys.Accession
;
import
org.genesys2.server.model.genesys.AccessionExchange
;
...
...
@@ -31,4 +32,7 @@ public interface AccessionExchangeRepository extends JpaRepository<AccessionExch
@Modifying
@Query
(
"delete from AccessionExchange ae where ae.accession.id in ?1"
)
void
deleteForAccessions
(
Collection
<
Long
>
accessionIds
);
@Query
(
"from AccessionExchange ae where ae.accession.id in ?1"
)
List
<
AccessionExchange
>
findAllFor
(
List
<
Long
>
accessionIds
);
}
src/main/java/org/genesys2/server/persistence/domain/AccessionGeoRepository.java
View file @
0abdbf0c
...
...
@@ -37,4 +37,7 @@ public interface AccessionGeoRepository extends JpaRepository<AccessionGeo, Long
@Query
(
"delete from AccessionGeo ag where ag.accession.id in ?1"
)
void
deleteForAccessions
(
Collection
<
Long
>
accessionIds
);
@Query
(
"from AccessionGeo ag where ag.accession.id in ?1"
)
List
<
AccessionGeo
>
findAllFor
(
List
<
Long
>
accessionIds
);
}
src/main/java/org/genesys2/server/persistence/domain/AccessionRemarkRepository.java
View file @
0abdbf0c
...
...
@@ -36,4 +36,7 @@ public interface AccessionRemarkRepository extends JpaRepository<AccessionRemark
@Modifying
@Query
(
"delete from AccessionRemark ar where ar.accession in (from Accession a where a.id in ( :ids ))"
)
void
deleteForAccessions
(
@Param
(
"ids"
)
Collection
<
Long
>
accessionIds
);
@Query
(
"from AccessionRemark ar where ar.accession.id in ?1"
)
List
<
AccessionRemark
>
findAllFor
(
List
<
Long
>
accessionIds
);
}
src/main/java/org/genesys2/server/service/GenesysService.java
View file @
0abdbf0c
...
...
@@ -103,7 +103,7 @@ public interface GenesysService {
Page
<
Object
[]>
statisticsGenusByInstitute
(
FaoInstitute
faoInstitute
,
Pageable
pageable
);
Page
<
Object
[]>
statisticsSpeciesByInstitute
(
FaoInstitute
faoInstitute
,
Pageable
pageable
);
Page
<
Object
[]>
statisticsSpeciesByInstitute
(
FaoInstitute
faoInstitute
,
Pageable
pageable
);
// Page<Object[]> statisticsCropByInstitute(FaoInstitute faoInstitute,
// Pageable pageable);
...
...
@@ -204,4 +204,20 @@ public interface GenesysService {
List
<
Long
>
listAccessionsIds
(
Taxonomy2
taxonomy
);
public
static
class
AllStuff
{
public
AllStuff
(
Long
id
)
{
this
.
id
=
id
;
}
public
Long
id
;
public
Accession
accession
=
null
;
public
AccessionGeo
geo
=
null
;
public
AccessionCollect
collect
=
null
;
public
AccessionBreeding
bred
=
null
;
public
List
<
AccessionAlias
>
names
=
null
;
public
AccessionExchange
exch
=
null
;
public
List
<
AccessionRemark
>
remarks
=
null
;
}
List
<
AllStuff
>
loadAllStuff
(
List
<
Long
>
accessionIds
);
}
src/main/java/org/genesys2/server/service/impl/DownloadServiceImpl.java
View file @
0abdbf0c
...
...
@@ -20,7 +20,7 @@ import java.io.IOException;
import
java.io.OutputStream
;
import
java.sql.ResultSet
;
import
java.sql.SQLException
;
import
java.util.
Collection
;
import
java.util.
ArrayList
;
import
java.util.Date
;
import
java.util.List
;
...
...
@@ -49,6 +49,7 @@ import org.genesys2.server.persistence.domain.GenesysLowlevelRepository;
import
org.genesys2.server.persistence.domain.MethodRepository
;
import
org.genesys2.server.service.DownloadService
;
import
org.genesys2.server.service.GenesysService
;
import
org.genesys2.server.service.GenesysService.AllStuff
;
import
org.genesys2.server.service.impl.FilterHandler.AppliedFilters
;
import
org.springframework.beans.factory.annotation.Autowired
;
import
org.springframework.beans.factory.annotation.Value
;
...
...
@@ -91,7 +92,6 @@ public class DownloadServiceImpl implements DownloadService {
dateStyle
.
setAlignment
(
CellStyle
.
ALIGN_LEFT
);
Sheet
legal
=
wb
.
getSheet
(
"Legal information"
);
System
.
err
.
println
(
"Rows: "
+
legal
.
getLastRowNum
());
Row
r
;
Cell
c
;
r
=
legal
.
createRow
(
0
);
...
...
@@ -117,65 +117,23 @@ public class DownloadServiceImpl implements DownloadService {
final
Sheet
sheet
=
wb
.
getSheet
(
"MCPD"
);
final
ArrayList
<
Long
>
batch
=
new
ArrayList
<
Long
>(
100
);
final
Writer
writer
=
new
Writer
();
// Write accession information
genesysLowlevelRepository
.
listAccessionIds
(
filters
,
new
Sort
(
"acceNumb"
),
new
RowCallbackHandler
()
{
int
i
=
0
;
@Override
public
void
processRow
(
ResultSet
rs
)
throws
SQLException
{
if
(
i
>
XLSX_ROWS_LIMIT
)
{
LOG
.
info
(
"Row limit exceeded"
);
return
;
}
long
accessionId
=
rs
.
getLong
(
1
);
Accession
accession
=
entityManager
.
find
(
Accession
.
class
,
accessionId
);
Row
row
=
sheet
.
createRow
(
i
+
1
);
AccessionGeo
geo
=
genesysService
.
listAccessionGeo
(
accession
);
AccessionCollect
collect
=
genesysService
.
listAccessionCollect
(
accession
);
AccessionBreeding
bred
=
genesysService
.
listAccessionBreeding
(
accession
);
List
<
AccessionAlias
>
names
=
genesysService
.
listAccessionAliases
(
accession
);
AccessionExchange
exch
=
genesysService
.
listAccessionExchange
(
accession
);
List
<
AccessionRemark
>
remarks
=
genesysService
.
listAccessionRemarks
(
accession
);
writeMCPDRow
(
sheet
,
row
,
accession
,
geo
,
collect
,
bred
,
names
,
exch
,
remarks
);
detach
(
accession
);
detach
(
geo
);
detach
(
collect
);
detach
(
bred
);
detach
(
names
);
detach
(
exch
);
detach
(
remarks
);
i
++;
if
(
i
%
100
==
0
&&
LOG
.
isDebugEnabled
())
{
LOG
.
debug
(
"Writing MCPD row "
+
i
);
}
if
(
i
%
1000
==
0
)
{
try
{
((
SXSSFSheet
)
sheet
).
flushRows
();
}
catch
(
IOException
e
)
{
LOG
.
error
(
e
.
getMessage
(),
e
);
}
}
}
private
void
detach
(
Object
obj
)
{
if
(
obj
!=
null
)
{
if
(
obj
instanceof
Collection
)
{
for
(
Object
o
:
(
Collection
<?>)
obj
)
detach
(
o
);
}
else
{
entityManager
.
detach
(
obj
);
}
if
(
batch
.
size
()
==
500
)
{
writer
.
processBatch
(
sheet
,
batch
);
batch
.
clear
();
}
batch
.
add
(
accessionId
);
}
});
writer
.
processBatch
(
sheet
,
batch
);
((
SXSSFSheet
)
sheet
).
flushRows
();
...
...
@@ -185,6 +143,33 @@ public class DownloadServiceImpl implements DownloadService {
LOG
.
info
(
"Done"
);
}
private
class
Writer
{
int
i
=
0
;
private
void
processBatch
(
Sheet
sheet
,
List
<
Long
>
batch
)
{
List
<
AllStuff
>
alls
=
genesysService
.
loadAllStuff
(
batch
);
for
(
AllStuff
all
:
alls
)
{
if
(
i
>
XLSX_ROWS_LIMIT
)
{
LOG
.
info
(
"Row limit exceeded"
);
return
;
}
Row
row
=
sheet
.
createRow
(
i
+
1
);
writeMCPDRow
(
sheet
,
row
,
all
.
accession
,
all
.
geo
,
all
.
collect
,
all
.
bred
,
all
.
names
,
all
.
exch
,
all
.
remarks
);
i
++;
}
LOG
.
info
(
"Writing MCPD row "
+
i
);
try
{
((
SXSSFSheet
)
sheet
).
flushRows
();
}
catch
(
IOException
e
)
{
LOG
.
error
(
e
.
getMessage
(),
e
);
}
}
}
private
void
writeMCPDRow
(
Sheet
sheet
,
Row
row
,
Accession
accession
,
AccessionGeo
geo
,
AccessionCollect
collect
,
AccessionBreeding
bred
,
List
<
AccessionAlias
>
names
,
AccessionExchange
exch
,
List
<
AccessionRemark
>
remarks
)
{
...
...
src/main/java/org/genesys2/server/service/impl/GenesysServiceImpl.java
View file @
0abdbf0c
...
...
@@ -26,6 +26,7 @@ import java.util.ArrayList;
import
java.util.Collection
;
import
java.util.Collections
;
import
java.util.Date
;
import
java.util.HashMap
;
import
java.util.HashSet
;
import
java.util.List
;
import
java.util.Map
;
...
...
@@ -33,6 +34,8 @@ import java.util.Set;
import
java.util.zip.ZipEntry
;
import
java.util.zip.ZipOutputStream
;
import
javax.persistence.EntityManager
;
import
org.apache.commons.collections.ListUtils
;
import
org.apache.commons.collections4.CollectionUtils
;
import
org.apache.commons.collections4.Predicate
;
...
...
@@ -112,6 +115,9 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
public
static
final
Log
LOG
=
LogFactory
.
getLog
(
GenesysServiceImpl
.
class
);
@Autowired
private
EntityManager
entityManager
;
// Services
@Autowired
private
TaxonomyService
taxonomyService
;
...
...
@@ -336,7 +342,7 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
for
(
Accession
accn
:
accessions
)
{
if
(
accn
==
null
)
continue
;
set
.
add
(
getAccessionDetails
(
accn
.
getId
()));
}
return
set
;
...
...
@@ -413,6 +419,87 @@ public class GenesysServiceImpl implements GenesysService, DatasetService {
return
accessionAliasRepository
.
findByAccession
(
accession
);
}
public
AllStuff
loadAllStuff
(
Accession
accession
)
{
if
(
accession
==
null
)
return
null
;
AllStuff
all
=
new
AllStuff
(
1
l
);
all
.
names
=
listAccessionAliases
(
accession
);
all
.
remarks
=
listAccessionRemarks
(
accession
);
return
all
;
}
@Override
public
List
<
AllStuff
>
loadAllStuff
(
List
<
Long
>
accessionIds
)
{
List
<
AllStuff
>
alls
=
new
ArrayList
<
AllStuff
>(
accessionIds
.
size
());
HashMap
<
Long
,
AllStuff
>
map
=
new
HashMap
<
Long
,
AllStuff
>();
for
(
Long
accessionId
:
accessionIds
)
{
AllStuff
all
=
new
AllStuff
(
accessionId
);
alls
.
add
(
all
);
map
.
put
(
accessionId
,
all
);
}
for
(
Accession
a
:
accessionRepository
.
findAll
(
accessionIds
))
{
AllStuff
all
=
map
.
get
(
a
.
getId
());
all
.
accession
=
a
;
a
.
getStoRage
().
size
();
all
.
names
=
new
ArrayList
<
AccessionAlias
>();
all
.
remarks
=
new
ArrayList
<
AccessionRemark
>();
}
for
(
AccessionCollect
c
:
accessionCollectRepository
.
findAllFor
(
accessionIds
))
{
AllStuff
all
=
map
.
get
(
c
.
getAccession
().
getId
());
all
.
collect
=
c
;
}
for
(
AccessionGeo
g
:
accessionGeoRepository
.
findAllFor
(
accessionIds
))
{
AllStuff
all
=
map
.
get
(
g
.
getAccession
().
getId
());
all
.
geo
=
g
;
}
for
(
AccessionBreeding
b
:
accessionBreedingRepository
.
findAllFor
(
accessionIds
))
{
AllStuff
all
=
map
.
get
(
b
.
getAccession
().
getId
());
all
.
bred
=
b
;
}
for
(
AccessionExchange
e
:
accessionExchangeRepository
.
findAllFor
(
accessionIds
))
{
AllStuff
all
=
map
.
get
(
e
.
getAccession
().
getId
());
all
.
exch
=
e
;
}
for
(
AccessionAlias
aa
:
accessionAliasRepository
.
findAllFor
(
accessionIds
))
{
AllStuff
all
=
map
.
get
(
aa
.
getAccession
().
getId
());
all
.
names
.
add
(
aa
);
}
for
(
AccessionRemark
ar
:
accessionRemarkRepository
.
findAllFor
(
accessionIds
))
{
AllStuff
all
=
map
.
get
(
ar
.
getAccession
().
getId
());
all
.
remarks
.
add
(
ar
);
}
for
(
AllStuff
all
:
alls
)
{
detach
(
all
.
accession
);
detach
(
all
.
geo
);
detach
(
all
.
collect
);
detach
(
all
.
bred
);
detach
(
all
.
names
);
detach
(
all
.
exch
);
detach
(
all
.
remarks
);
}
return
alls
;
}
private
void
detach
(
Object
obj
)
{
if
(
obj
!=
null
)
{
if
(
obj
instanceof
Collection
)
{
for
(
Object
o
:
(
Collection
<?>)
obj
)
detach
(
o
);
}
else
{
entityManager
.
detach
(
obj
);
}
}
}
@Override
public
List
<
Metadata
>
listMetadata
(
Accession
accession
)
{
final
List
<
Long
>
x
=
accessionTraitRepository
.
listMetadataIds
(
accession
);
...
...
src/main/resources/template/download/MCPD.xlsx
View file @
0abdbf0c
No preview for this file type
src/main/webapp/WEB-INF/jsp/selection/index.jsp
View file @
0abdbf0c
...
...
@@ -21,21 +21,26 @@
<div
class=
"main-col-header clearfix"
>
<div
class=
"nav-header"
>
<form
class=
"pull-right form-horizontal"
method=
"post"
action=
"
<c:url
value=
"/sel/dwca"
/>
"
>
<input
type=
"hidden"
name=
"${_csrf.parameterName}"
value=
"${_csrf.token}"
/>
<div
class=
"row"
style=
"margin-top: 2em;"
>
<div
class=
"col-sm-4"
>
<button
class=
"btn btn-default"
type=
"submit"
><spring:message
code=
"filter.download-dwca"
/></button>
<div
class=
"results"
><spring:message
code=
"accessions.number"
arguments=
"
${
pagedData
.
totalElements
}
"
/></div>
<div
class=
"row"
>
<div
class=
"col-sm-12 col-md-6"
>
<div
class=
"pagination"
>
<spring:message
code=
"paged.pageOfPages"
arguments=
"
${
pagedData
.
number
+
1
}
,${pagedData.totalPages}"
/>
<a
href=
"
<spring:url
value=
""
><spring:param
name=
"page"
value=
"
${
pagedData
.
number
eq
0
?
1
:
pagedData
.
number
}
"
/><spring:param
name=
"filter"
value=
"
${
jsonFilter
}
"
/></spring:url>
"
><spring:message
code=
"pagination.previous-page"
/></a>
<a
href=
"
<spring:url
value=
""
><spring:param
name=
"page"
value=
"
${
pagedData
.
number
+
2
}
"
/><spring:param
name=
"filter"
value=
"
${
jsonFilter
}
"
/></spring:url>
"
><spring:message
code=
"pagination.next-page"
/></a>
</div>
</div>
</form>
<div
class=
"results"
><spring:message
code=
"accessions.number"
arguments=
"
${
pagedData
.
totalElements
}
"
/></div>
<div
class=
"pagination"
>
<spring:message
code=
"paged.pageOfPages"
arguments=
"
${
pagedData
.
number
+
1
}
,${pagedData.totalPages}"
/>
<a
href=
"
<spring:url
value=
""
><spring:param
name=
"page"
value=
"
${
pagedData
.
number
eq
0
?
1
:
pagedData
.
number
}
"
/><spring:param
name=
"filter"
value=
"
${
jsonFilter
}
"
/></spring:url>
"
><spring:message
code=
"pagination.previous-page"
/></a>
<a
href=
"
<spring:url
value=
""
><spring:param
name=
"page"
value=
"
${
pagedData
.
number
+
2
}
"
/><spring:param
name=
"filter"
value=
"
${
jsonFilter
}
"
/></spring:url>
"
><spring:message
code=
"pagination.next-page"
/></a>
<div
class=
"col-sm-12 col-md-6"
style=
"text-align: right; padding-top: 12px"
>
<form
style=
"display: inline-block"
method=
"post"
action=
"
<c:url
value=
"/sel/dwca"
/>
"
>
<input
type=
"hidden"
name=
"${_csrf.parameterName}"
value=
"${_csrf.token}"
/>
<button
class=
"btn btn-default"
type=
"submit"
><spring:message
code=
"filter.download-dwca"
/></button>
</form>
<form
style=
"display: inline-block"
method=
"post"
action=
"
<c:url
value=
"/sel/download/mcpd"
/>
"
>
<input
type=
"hidden"
name=
"${_csrf.parameterName}"
value=
"${_csrf.token}"
/>
<button
class=
"btn btn-default"
type=
"submit"
><spring:message
code=
"filter.download-mcpd"
/></button>
</form>
</div>
</div>
</div>
</div>
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment