Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Genesys Backend
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
44
Issues
44
List
Boards
Labels
Service Desk
Milestones
Operations
Operations
Incidents
Packages & Registries
Packages & Registries
Container Registry
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
Genesys PGR
Genesys Backend
Commits
4cdf9e55
Commit
4cdf9e55
authored
Jan 14, 2019
by
Matija Obreza
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Download worldclim.org dataset as Excel
parent
553f019e
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
90 additions
and
49 deletions
+90
-49
src/main/java/org/genesys2/server/mvc/admin/DS2Controller.java
...ain/java/org/genesys2/server/mvc/admin/DS2Controller.java
+25
-0
src/main/java/org/genesys2/server/service/impl/DSServiceImpl.java
.../java/org/genesys2/server/service/impl/DSServiceImpl.java
+58
-49
src/main/webapp/WEB-INF/jsp/admin/ds2/index.jsp
src/main/webapp/WEB-INF/jsp/admin/ds2/index.jsp
+7
-0
No files found.
src/main/java/org/genesys2/server/mvc/admin/DS2Controller.java
View file @
4cdf9e55
...
...
@@ -16,7 +16,11 @@
package
org.genesys2.server.mvc.admin
;
import
java.io.EOFException
;
import
java.io.IOException
;
import
java.io.OutputStream
;
import
javax.servlet.http.HttpServletResponse
;
import
org.genesys2.server.model.dataset.DS
;
import
org.genesys2.server.model.dataset.DSColumn
;
...
...
@@ -68,4 +72,25 @@ public class DS2Controller {
}
return
"redirect:/admin/ds2/"
;
}
@RequestMapping
(
value
=
"/worldclim/download"
,
method
=
RequestMethod
.
POST
)
public
void
worldclimDownload
(
HttpServletResponse
response
)
throws
IOException
{
DS
ds
=
dsService
.
loadDatasetByUuid
(
WorldClimUpdater
.
WORLDCLIM_DATASET
);
// Write MCPD to the stream.
response
.
setContentType
(
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
);
response
.
addHeader
(
"Content-Disposition"
,
String
.
format
(
"attachment; filename=\"worldclim-%1s.xlsx\""
,
System
.
currentTimeMillis
()));
// response.flushBuffer();
final
OutputStream
outputStream
=
response
.
getOutputStream
();
try
{
dsService
.
download
(
ds
,
outputStream
);
response
.
flushBuffer
();
}
catch
(
EOFException
e
)
{
LOG
.
warn
(
"Download was aborted: {}"
,
e
.
getMessage
());
}
}
}
src/main/java/org/genesys2/server/service/impl/DSServiceImpl.java
View file @
4cdf9e55
...
...
@@ -18,8 +18,10 @@ import java.util.regex.Pattern;
import
org.apache.poi.ss.usermodel.Cell
;
import
org.apache.poi.ss.usermodel.Row
;
import
org.apache.poi.xssf.streaming.SXSSFSheet
;
import
org.apache.poi.xssf.streaming.SXSSFWorkbook
;
import
org.apache.poi.xssf.streaming.SXSSFRow
;
import
org.apache.poi.xssf.streaming.SuperSXSSFSheet
;
import
org.apache.poi.xssf.streaming.SuperSXSSFWorkbook
;
import
org.apache.poi.xssf.usermodel.XSSFSheet
;
import
org.apache.poi.xssf.usermodel.XSSFWorkbook
;
import
org.genesys2.server.model.dataset.DS
;
import
org.genesys2.server.model.dataset.DSColumn
;
...
...
@@ -413,29 +415,32 @@ public class DSServiceImpl implements DSService {
@Override
public
void
download
(
DS
ds
,
List
<
DSColumn
>
dsds
,
OutputStream
outputStream
)
throws
IOException
{
XSSFWorkbook
template
=
new
XSSFWorkbook
();
// keep 1000 rows in memory, exceeding rows will be flushed to disk
SXSSFWorkbook
wb
=
new
SXSSFWorkbook
(
template
,
100
);
SXSSFSheet
sheet
=
(
SXSSFSheet
)
wb
.
createSheet
(
"worldclim.org"
);
int
rowIndex
=
0
,
cellIndex
=
0
;
Row
r
=
sheet
.
createRow
(
rowIndex
++);
r
.
createCell
(
cellIndex
++).
setCellValue
(
"row_id"
);
for
(
DSQualifier
dsq
:
ds
.
getQualifiers
())
{
r
.
createCell
(
cellIndex
++).
setCellValue
(
dsq
.
getDescriptor
().
getCode
());
}
// Prepare descriptor mapping
Long
[]
columnDescriptors
=
new
Long
[
dsds
.
size
()];
XSSFWorkbook
template
=
new
XSSFWorkbook
();
{
int
columnIndex
=
0
;
for
(
DSColumn
dsd
:
dsds
)
{
r
.
createCell
(
cellIndex
++).
setCellValue
(
dsd
.
getDescriptor
().
getCode
());
columnDescriptors
[
columnIndex
++]
=
dsd
.
getId
();
XSSFSheet
sheet
=
template
.
createSheet
(
"worldclim.org"
);
int
rowIndex
=
0
,
cellIndex
=
0
;
Row
r
=
sheet
.
createRow
(
rowIndex
++);
r
.
createCell
(
cellIndex
++).
setCellValue
(
"row_id"
);
for
(
DSQualifier
dsq
:
ds
.
getQualifiers
())
{
r
.
createCell
(
cellIndex
++).
setCellValue
(
dsq
.
getDescriptor
().
getCode
());
}
// Prepare descriptor mapping
{
int
columnIndex
=
0
;
for
(
DSColumn
dsd
:
dsds
)
{
r
.
createCell
(
cellIndex
++).
setCellValue
(
dsd
.
getDescriptor
().
getCode
());
columnDescriptors
[
columnIndex
++]
=
dsd
.
getId
();
}
}
}
// keep 1000 rows in memory, exceeding rows will be flushed to disk
SuperSXSSFWorkbook
wb
=
new
SuperSXSSFWorkbook
(
template
,
50
);
SuperSXSSFSheet
sheet
=
wb
.
getStreamingSheet
(
"worldclim.org"
);
// List<DSRow> rows = dsRowRepo.findByDataset(ds);
// LOG.info("Retrieved DS rows " +rows.size());
...
...
@@ -443,43 +448,47 @@ public class DSServiceImpl implements DSService {
List
<
Object
[]>
allDsrq
=
dsRowRepo
.
getQualifiers
(
ds
);
LOG
.
info
(
"Got {} row qualifiers"
,
allDsrq
.
size
());
int
batchSize
=
100
;
for
(
int
fromIndex
=
0
;
fromIndex
<
allDsrq
.
size
();
fromIndex
+=
batchSize
)
{
List
<
Object
[]>
batch
=
allDsrq
.
subList
(
fromIndex
,
Math
.
min
(
fromIndex
+
batchSize
,
allDsrq
.
size
()));
LOG
.
info
(
"Processing position: {} of {}"
,
fromIndex
,
allDsrq
.
size
());
List
<
Long
>
rowIds
=
new
ArrayList
<
Long
>(
batchSize
);
for
(
Object
[]
x
:
batch
)
{
rowIds
.
add
((
Long
)
x
[
0
]);
}
List
<
Object
[]>
allValues
=
dsRowRepo
.
getRowValues
(
rowIds
,
columnDescriptors
);
int
batchPos
=
0
;
for
(
Object
[]
x
:
batch
)
{
r
=
sheet
.
createRow
(
rowIndex
++);
cellIndex
=
0
;
for
(
Object
v
:
x
)
{
addCell
(
r
,
cellIndex
++,
v
);
sheet
.
setRowGenerator
((
sheet2
)
->
{
int
batchSize
=
100
;
int
rowIndex
=
1
,
cellIndex
=
0
;
for
(
int
fromIndex
=
0
;
fromIndex
<
allDsrq
.
size
();
fromIndex
+=
batchSize
)
{
List
<
Object
[]>
batch
=
allDsrq
.
subList
(
fromIndex
,
Math
.
min
(
fromIndex
+
batchSize
,
allDsrq
.
size
()));
LOG
.
info
(
"Processing position: {} of {}"
,
fromIndex
,
allDsrq
.
size
());
List
<
Long
>
rowIds
=
new
ArrayList
<
Long
>(
batchSize
);
for
(
Object
[]
x
:
batch
)
{
rowIds
.
add
((
Long
)
x
[
0
]);
}
Object
[]
values
=
allValues
.
get
(
batchPos
++);
if
(
values
!=
null
)
{
for
(
Object
v
:
values
)
{
List
<
Object
[]>
allValues
=
dsRowRepo
.
getRowValues
(
rowIds
,
columnDescriptors
);
int
batchPos
=
0
;
for
(
Object
[]
x
:
batch
)
{
SXSSFRow
r
=
sheet
.
createRow
(
rowIndex
++);
cellIndex
=
0
;
for
(
Object
v
:
x
)
{
addCell
(
r
,
cellIndex
++,
v
);
}
Object
[]
values
=
allValues
.
get
(
batchPos
++);
if
(
values
!=
null
)
{
for
(
Object
v
:
values
)
{
addCell
(
r
,
cellIndex
++,
v
);
}
}
}
// if (rowIndex > 10000) {
// LOG.warn("Breaking");
// break;
// }
}
// if (rowIndex > 10000) {
// LOG.warn("Breaking");
// break;
// }
}
sheet
.
flushRows
();
});
LOG
.
info
(
"Writing to output stream"
);
wb
.
write
(
outputStream
);
wb
.
close
();
wb
.
dispose
();
outputStream
.
flush
();
LOG
.
info
(
"Done"
);
}
...
...
src/main/webapp/WEB-INF/jsp/admin/ds2/index.jsp
View file @
4cdf9e55
...
...
@@ -15,12 +15,19 @@
<input
type=
"hidden"
name=
"${_csrf.parameterName}"
value=
"${_csrf.token}"
/>
</form>
<form
method=
"post"
action=
"
<c:url
value=
"/admin/ds2/worldclim/download"
/>
"
>
<input
type=
"submit"
class=
"btn btn-default"
value=
"Download worldclim.org"
/>
<!-- CSRF protection -->
<input
type=
"hidden"
name=
"${_csrf.parameterName}"
value=
"${_csrf.token}"
/>
</form>
<form
method=
"post"
action=
"
<c:url
value=
"/admin/ds2/worldclim/delete"
/>
"
>
<input
type=
"submit"
class=
"btn btn-default"
value=
"Delete worldclim.org"
/>
<!-- CSRF protection -->
<input
type=
"hidden"
name=
"${_csrf.parameterName}"
value=
"${_csrf.token}"
/>
</form>
<content
tag=
"javascript"
>
<script
type=
"text/javascript"
>
$
(
document
).
ready
(
function
()
{
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment