Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Genesys PGR
Genesys Backend
Commits
2509c2d8
Commit
2509c2d8
authored
Mar 05, 2015
by
Matija Obreza
Browse files
Downloading + Hibernate Stateless test
parent
62df87bd
Changes
9
Hide whitespace changes
Inline
Side-by-side
src/main/java/org/genesys2/server/model/dataset/DSValue.java
View file @
2509c2d8
...
...
@@ -3,6 +3,7 @@ package org.genesys2.server.model.dataset;
import
javax.persistence.DiscriminatorColumn
;
import
javax.persistence.DiscriminatorType
;
import
javax.persistence.Entity
;
import
javax.persistence.FetchType
;
import
javax.persistence.GeneratedValue
;
import
javax.persistence.GenerationType
;
import
javax.persistence.Id
;
...
...
@@ -22,11 +23,11 @@ public abstract class DSValue<T> {
@GeneratedValue
(
strategy
=
GenerationType
.
TABLE
)
private
Long
id
;
@ManyToOne
(
optional
=
false
)
@ManyToOne
(
fetch
=
FetchType
.
LAZY
,
optional
=
false
)
@JoinColumn
(
name
=
"r"
)
private
DSRow
row
;
@ManyToOne
(
optional
=
false
)
@ManyToOne
(
fetch
=
FetchType
.
LAZY
,
optional
=
false
)
@JoinColumn
(
name
=
"dsd"
)
private
DSDescriptor
datasetDescriptor
;
...
...
src/main/java/org/genesys2/server/persistence/domain/dataset/DSRowCustomRepository.java
View file @
2509c2d8
package
org.genesys2.server.persistence.domain.dataset
;
import
java.util.Collection
;
import
java.util.List
;
import
java.util.Map
;
import
org.genesys2.server.model.dataset.DS
;
...
...
@@ -13,4 +14,25 @@ public interface DSRowCustomRepository {
DSRow
findRow
(
DS
ds
,
Object
[]
qualifiers
);
Map
<
Long
,
DSValue
<?>>
rowValueMap
(
Collection
<
DSRow
>
dsrs
,
DSDescriptor
dsd
);
List
<
Object
[]>
getQualifiers
(
DS
ds
);
/**
* Array of dsv.values as per DSDescriptor order
*
* @param rowId
* @param columnDescriptors
* @return
*/
Object
[]
getRowValues
(
long
rowId
,
Long
[]
columnDescriptors
);
/**
* List of arrays of dsv.values as per rowId order and DSDescriptor order
*
* @param rowId
* @param columnDescriptors
* @return
*/
List
<
Object
[]>
getRowValues
(
List
<
Long
>
rowId
,
Long
[]
columnDescriptors
);
}
src/main/java/org/genesys2/server/persistence/domain/dataset/DSRowRepository.java
View file @
2509c2d8
...
...
@@ -20,4 +20,7 @@ public interface DSRowRepository extends JpaRepository<DSRow, Long>, DSRowCustom
@Query
(
"select dsrq.row from DSRowQualifierLong dsrq where dsrq.datasetQualifier.dataset = ?1 and dsrq.value in ( ?2 )"
)
List
<
DSRow
>
findByLongQualifier
(
DS
ds
,
Collection
<?>
qualifiers
);
@Query
(
"select distinct dsrq.row from DSRowQualifier dsrq where dsrq.datasetQualifier.dataset = ?1"
)
List
<
DSRow
>
findByDataset
(
DS
ds
);
}
src/main/java/org/genesys2/server/persistence/domain/dataset/DSRowRepositoryCustomImpl.java
View file @
2509c2d8
...
...
@@ -16,8 +16,14 @@ import org.apache.commons.logging.Log;
import
org.apache.commons.logging.LogFactory
;
import
org.genesys2.server.model.dataset.DS
;
import
org.genesys2.server.model.dataset.DSDescriptor
;
import
org.genesys2.server.model.dataset.DSQualifier
;
import
org.genesys2.server.model.dataset.DSRow
;
import
org.genesys2.server.model.dataset.DSValue
;
import
org.hibernate.ScrollMode
;
import
org.hibernate.ScrollableResults
;
import
org.hibernate.SessionFactory
;
import
org.hibernate.StatelessSession
;
import
org.hibernate.Transaction
;
import
org.springframework.stereotype.Repository
;
@Repository
...
...
@@ -95,7 +101,121 @@ public class DSRowRepositoryCustomImpl implements DSRowCustomRepository {
map
.
put
((
Long
)
r
[
0
],
(
DSValue
<?>)
r
[
1
]);
}
}
return
map
;
}
/**
* Return row.id, dsdq1.value, dsdq2.value...
*/
@SuppressWarnings
(
"unchecked"
)
@Override
public
List
<
Object
[]>
getQualifiers
(
DS
ds
)
{
StringBuilder
select
=
new
StringBuilder
();
StringBuilder
where
=
new
StringBuilder
();
StringBuilder
from
=
new
StringBuilder
();
List
<
DSQualifier
>
dsq
=
ds
.
getQualifiers
();
Class
<?>
clazz
;
final
int
OFFSET
=
1
;
int
paramIdx
=
OFFSET
;
for
(
int
i
=
0
;
i
<
dsq
.
size
();
i
++)
{
if
(
i
>
0
)
from
.
append
(
", "
);
// TODO Get from descriptor
clazz
=
Long
.
class
;
if
(
clazz
.
equals
(
Long
.
class
))
{
from
.
append
(
"DSRowQualifierLong dsrq"
).
append
(
i
);
}
else
if
(
clazz
.
equals
(
Double
.
class
))
{
from
.
append
(
"DSRowQualifierDouble dsrq"
).
append
(
i
);
}
else
if
(
clazz
.
equals
(
String
.
class
))
{
from
.
append
(
"DSRowQualifierString dsrq"
).
append
(
i
);
}
select
.
append
(
", "
).
append
(
"dsrq"
).
append
(
i
).
append
(
".value"
);
if
(
i
>
0
)
{
where
.
append
(
" and dsrq"
+
(
i
-
1
)
+
".row = dsrq"
+
i
+
".row and "
);
}
where
.
append
(
" dsrq"
+
i
+
".datasetQualifier = ?"
).
append
(
paramIdx
++);
}
LOG
.
info
(
"select dsrq0.row from "
+
from
+
" where "
+
where
.
toString
());
Query
q
=
entityManager
.
createQuery
(
"select dsrq0.row.id "
+
select
+
" from "
+
from
+
" where "
+
where
.
toString
()
+
""
);
paramIdx
=
OFFSET
;
for
(
int
i
=
0
;
i
<
dsq
.
size
();
i
++)
{
q
.
setParameter
(
paramIdx
++,
dsq
);
}
return
q
.
getResultList
();
}
@Override
public
Object
[]
getRowValues
(
long
rowId
,
Long
[]
columnDescriptors
)
{
Query
q
=
entityManager
.
createQuery
(
"select dsrv.datasetDescriptor.id, dsrv from DSValue dsrv where dsrv.row.id = ?1"
);
q
.
setParameter
(
1
,
rowId
);
// List of [dsd.id, dsrv]
@SuppressWarnings
(
"unchecked"
)
List
<
Object
[]>
values
=
q
.
getResultList
();
Object
[]
result
=
new
Object
[
columnDescriptors
.
length
];
for
(
Object
[]
v
:
values
)
{
Long
dsdId
=
(
Long
)
v
[
0
];
for
(
int
i
=
0
;
i
<
columnDescriptors
.
length
;
i
++)
{
if
(
columnDescriptors
[
i
].
equals
(
dsdId
))
{
result
[
i
]
=
((
DSValue
<?>)
v
[
1
]).
getValue
();
}
}
}
return
result
;
}
@Override
public
List
<
Object
[]>
getRowValues
(
List
<
Long
>
rowIds
,
Long
[]
columnDescriptors
)
{
// LOG.info("Making stateless");
SessionFactory
sessionFactory
=
entityManager
.
getEntityManagerFactory
().
unwrap
(
SessionFactory
.
class
);
StatelessSession
session
=
sessionFactory
.
openStatelessSession
();
Transaction
tx
=
session
.
beginTransaction
();
org
.
hibernate
.
Query
q
=
session
.
createQuery
(
"select dsrv.datasetDescriptor.id, dsrv.row.id, dsrv from DSValue dsrv where dsrv.row.id in (:ids)"
);
q
.
setParameterList
(
"ids"
,
rowIds
);
// LOG.info("Query ready");
// System.out.println("-");
// Initialize results list with blank arrays
List
<
Object
[]>
results
=
new
ArrayList
<
Object
[]>(
rowIds
.
size
());
for
(
int
i
=
0
;
i
<
rowIds
.
size
();
i
++)
{
results
.
add
(
new
Object
[
columnDescriptors
.
length
]);
}
// List of [dsd.id, dsrv]
ScrollableResults
scroll
=
q
.
scroll
(
ScrollMode
.
FORWARD_ONLY
);
// LOG.info("Scrolling");
int
entries
=
0
;
while
(
scroll
.
next
())
{
entries
++;
Object
[]
v
=
scroll
.
get
();
// LOG.info("Got next");
Long
dsdId
=
(
Long
)
v
[
0
];
Long
rowId
=
(
Long
)
v
[
1
];
DSValue
<?>
dsv
=
(
DSValue
<?>)
v
[
2
];
Object
[]
result
=
results
.
get
(
rowIds
.
indexOf
(
rowId
));
for
(
int
i
=
0
;
i
<
columnDescriptors
.
length
;
i
++)
{
if
(
columnDescriptors
[
i
].
equals
(
dsdId
))
{
result
[
i
]
=
dsv
.
getValue
();
}
}
}
// LOG.info("Done scrolling entries=" + entries);
tx
.
commit
();
session
.
close
();
// LOG.info("Closing session");
return
results
;
}
}
src/main/java/org/genesys2/server/service/DSService.java
View file @
2509c2d8
package
org.genesys2.server.service
;
import
java.io.BufferedOutputStream
;
import
java.io.IOException
;
import
java.io.OutputStream
;
import
java.nio.MappedByteBuffer
;
import
java.util.List
;
import
java.util.Set
;
...
...
@@ -31,4 +34,6 @@ public interface DSService {
void
tempStore
(
List
<
AccessionGeo
>
toSave
);
WorldclimJson
jsonForTile
(
DS
worldClimDataset
,
Long
tileIndex
);
void
download
(
DS
ds
,
OutputStream
fos
)
throws
IOException
;
}
src/main/java/org/genesys2/server/service/impl/DSServiceImpl.java
View file @
2509c2d8
package
org.genesys2.server.service.impl
;
import
java.io.IOException
;
import
java.io.OutputStream
;
import
java.nio.ByteBuffer
;
import
java.nio.MappedByteBuffer
;
import
java.nio.charset.Charset
;
import
java.util.ArrayList
;
import
java.util.Collection
;
import
java.util.Date
;
import
java.util.HashMap
;
import
java.util.HashSet
;
import
java.util.Iterator
;
...
...
@@ -18,6 +21,11 @@ import java.util.regex.Pattern;
import
org.apache.commons.codec.digest.DigestUtils
;
import
org.apache.commons.logging.Log
;
import
org.apache.commons.logging.LogFactory
;
import
org.apache.poi.ss.usermodel.Cell
;
import
org.apache.poi.ss.usermodel.Row
;
import
org.apache.poi.xssf.streaming.SXSSFSheet
;
import
org.apache.poi.xssf.streaming.SXSSFWorkbook
;
import
org.apache.poi.xssf.usermodel.XSSFWorkbook
;
import
org.genesys2.server.model.dataset.DS
;
import
org.genesys2.server.model.dataset.DSDescriptor
;
import
org.genesys2.server.model.dataset.DSQualifier
;
...
...
@@ -264,7 +272,7 @@ public class DSServiceImpl implements DSService {
missing
.
remove
(
dsr
.
getRowQualifiers
().
get
(
0
).
getValue
());
}
for
(
Long
miss
:
missing
)
{
//
LOG.info("Adding missing row for " + miss);
//
LOG.info("Adding missing row for " + miss);
dsrs
.
add
(
makeRow
(
ds
,
miss
));
}
}
...
...
@@ -421,4 +429,90 @@ public class DSServiceImpl implements DSService {
}
return
wc
;
}
@Override
public
void
download
(
DS
ds
,
OutputStream
outputStream
)
throws
IOException
{
XSSFWorkbook
template
=
new
XSSFWorkbook
();
// keep 1000 rows in memory, exceeding rows will be flushed to disk
SXSSFWorkbook
wb
=
new
SXSSFWorkbook
(
template
,
1000
);
SXSSFSheet
sheet
=
(
SXSSFSheet
)
wb
.
createSheet
(
"worldclim.org"
);
int
rowIndex
=
0
,
cellIndex
=
0
;
Row
r
=
sheet
.
createRow
(
rowIndex
++);
r
.
createCell
(
cellIndex
++).
setCellValue
(
"row_id"
);
for
(
DSQualifier
dsq
:
ds
.
getQualifiers
())
{
r
.
createCell
(
cellIndex
++).
setCellValue
(
dsq
.
getDescriptor
().
getCode
());
}
// Prepare descriptor mapping
Long
[]
columnDescriptors
=
new
Long
[
ds
.
getDescriptors
().
size
()];
{
int
columnIndex
=
0
;
for
(
DSDescriptor
dsd
:
ds
.
getDescriptors
())
{
r
.
createCell
(
cellIndex
++).
setCellValue
(
dsd
.
getDescriptor
().
getCode
());
columnDescriptors
[
columnIndex
++]
=
dsd
.
getId
();
}
}
// List<DSRow> rows = dsRowRepo.findByDataset(ds);
// LOG.info("Retrieved DS rows " +rows.size());
// rowId, dsdq1.value, dsdq2.value...
List
<
Object
[]>
allDsrq
=
dsRowRepo
.
getQualifiers
(
ds
);
LOG
.
info
(
"Got "
+
allDsrq
.
size
()
+
" row qualifiers"
);
int
batchSize
=
100
;
for
(
int
fromIndex
=
0
;
fromIndex
<
allDsrq
.
size
();
fromIndex
+=
batchSize
)
{
List
<
Object
[]>
batch
=
allDsrq
.
subList
(
fromIndex
,
Math
.
min
(
fromIndex
+
batchSize
,
allDsrq
.
size
()));
LOG
.
info
(
"Processing position: "
+
fromIndex
+
" of "
+
allDsrq
.
size
());
List
<
Long
>
rowIds
=
new
ArrayList
<
Long
>(
batchSize
);
for
(
Object
[]
x
:
batch
)
{
rowIds
.
add
((
Long
)
x
[
0
]);
}
List
<
Object
[]>
allValues
=
dsRowRepo
.
getRowValues
(
rowIds
,
columnDescriptors
);
int
batchPos
=
0
;
for
(
Object
[]
x
:
batch
)
{
r
=
sheet
.
createRow
(
rowIndex
++);
cellIndex
=
0
;
for
(
Object
v
:
x
)
{
addCell
(
r
,
cellIndex
++,
v
);
}
Object
[]
values
=
allValues
.
get
(
batchPos
++);
if
(
values
!=
null
)
{
for
(
Object
v
:
values
)
{
addCell
(
r
,
cellIndex
++,
v
);
}
}
}
if
(
rowIndex
>
10000
)
{
LOG
.
warn
(
"Breaking"
);
break
;
}
}
sheet
.
flushRows
();
LOG
.
info
(
"Writing to output stream"
);
wb
.
write
(
outputStream
);
wb
.
dispose
();
LOG
.
info
(
"Done"
);
}
private
void
addCell
(
Row
r
,
int
cellIndex
,
Object
v
)
{
if
(
v
==
null
)
return
;
Cell
c
=
r
.
createCell
(
cellIndex
);
if
(
v
instanceof
String
)
{
c
.
setCellValue
((
String
)
v
);
}
else
if
(
v
instanceof
Number
)
{
c
.
setCellValue
(((
Number
)
v
).
doubleValue
());
}
else
if
(
v
instanceof
Date
)
{
c
.
setCellValue
((
Date
)
v
);
}
}
}
src/main/java/org/genesys2/server/service/worker/WorldClimUpdater.java
View file @
2509c2d8
...
...
@@ -77,7 +77,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
public
class
WorldClimUpdater
implements
InitializingBean
{
public
static
final
Log
LOG
=
LogFactory
.
getLog
(
WorldClimUpdater
.
class
);
p
rivate
static
final
UUID
WORLDCLIM_DATASET
=
UUID
.
fromString
(
"BC84433B-A626-4BDF-97D3-DB36D79499C6"
);
p
ublic
static
final
UUID
WORLDCLIM_DATASET
=
UUID
.
fromString
(
"BC84433B-A626-4BDF-97D3-DB36D79499C6"
);
public
static
String
WORLDCLIM_ORG
=
"http://www.worldclim.org"
;
public
static
String
WORLDCLIM_ORG_FILEBASE
=
"http://biogeo.ucdavis.edu/data/climate/worldclim/1_4/grid/cur/"
;
...
...
src/test/java/org/genesys2/server/service/impl/DSTest.java
View file @
2509c2d8
...
...
@@ -35,7 +35,7 @@ import com.hazelcast.spring.cache.HazelcastCacheManager;
@ActiveProfiles
(
"dev"
)
public
class
DSTest
{
@Import
({
HazelcastConfig
.
class
,
Jpa
Real
DataConfig
.
class
})
@Import
({
HazelcastConfig
.
class
,
JpaDataConfig
.
class
})
public
static
class
Config
{
@Bean
...
...
src/test/java/org/genesys2/server/service/impl/WorldClimUpdaterTest.java
View file @
2509c2d8
package
org.genesys2.server.service.impl
;
import
java.io.BufferedOutputStream
;
import
java.io.File
;
import
java.io.FileOutputStream
;
import
java.io.IOException
;
import
org.apache.http.impl.client.HttpClientBuilder
;
import
org.genesys2.server.model.dataset.DS
;
import
org.genesys2.server.persistence.domain.GenesysLowlevelRepository
;
import
org.genesys2.server.persistence.domain.GenesysLowlevelRepositoryCustomImpl
;
import
org.genesys2.server.service.DSService
;
...
...
@@ -67,6 +71,9 @@ public class WorldClimUpdaterTest {
@Autowired
private
WorldClimUpdater
worldClimUpdater
;
@Autowired
private
DSService
dsService
;
@Test
public
void
testUpdateAlt
()
throws
IOException
{
worldClimUpdater
.
update
(
"alt"
);
...
...
@@ -83,4 +90,14 @@ public class WorldClimUpdaterTest {
}
}
@Test
public
void
testDownload
()
throws
IOException
{
DS
ds
=
dsService
.
loadDatasetByUuid
(
WorldClimUpdater
.
WORLDCLIM_DATASET
);
File
file
=
new
File
(
"worldclim "
+
System
.
currentTimeMillis
()
+
".xls"
);
BufferedOutputStream
fos
=
new
BufferedOutputStream
(
new
FileOutputStream
(
file
));
dsService
.
download
(
ds
,
fos
);
fos
.
flush
();
fos
.
close
();
// file.delete();
}
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment