Skip to content

Commit 9ebdaaa

Browse files
malagsmalags
authored
Merge pull request #86 from LuCEresearchlab/push-new-dataset-better-handling
Push new dataset better handling
2 parents be431bb + ecda833 commit 9ebdaaa

23 files changed

+302
-201
lines changed

‎.env

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,4 @@
11
FLASK_APP=flaskr
22
FLASK_ENV=development
3+
NR_WORKERS=2
4+
NR_THREADS=5

‎docker-compose.yml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,14 @@ services:
2222
environment:
2323
- FLASK_APP=${FLASK_APP}
2424
- FLASK_ENV=${FLASK_ENV}
25+
- NR_WORKERS=${NR_WORKERS}
26+
- NR_THREADS=${NR_THREADS}
2527
depends_on:
2628
- tagging-database
2729

2830

2931
tagging-database:
30-
image: 'mongo:4.4.3'
32+
image: 'mongo:4.2.13-bionic'
3133
ports:
3234
- "27017:27017"
3335
volumes:

‎frontend/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM node:15.12-alpine3.10
1+
FROM node:15.14.0-alpine3.10
22

33
EXPOSE 8080
44

‎frontend/src/components/v2/TaggingUI.tsx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ function TaggingUI({taggingSession, dispatchTaggingSession, taggingClusterSessio
7171
}
7272

7373
const clusterFetch = useFetch<any>(
74-
`${TAGGING_SERVICE_URL}/datasets/clusters/dataset/${taggingClusterSession.dataset_id
74+
`${TAGGING_SERVICE_URL}/clusters/dataset/${taggingClusterSession.dataset_id
7575
}/question/${taggingClusterSession.question_id
7676
}/user/${taggingClusterSession.user_id}`)
7777

‎frontend/src/helpers/PostHelper.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,6 @@ export function postClusters(dataset_id: string,
4545
question_id: string,
4646
user_id: string,
4747
clusters: Answer[][]) {
48-
post(`${TAGGING_SERVICE_URL}/datasets/clusters/dataset/${dataset_id}/question/${question_id}/user/${user_id}`,
48+
post(`${TAGGING_SERVICE_URL}/clusters/dataset/${dataset_id}/question/${question_id}/user/${user_id}`,
4949
clusters)
5050
}

‎frontend/src/index.html

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
<html>
33
<head>
44
<meta charset="utf-8" />
5-
<title>My app</title>
5+
<title>TSR</title>
66
</head>
77
<body style="margin:0;">
88
<div id="root"></div>

‎frontend/src/interfaces/Dataset.ts

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,18 @@
11
export interface Dataset {
2-
name: string,
2+
"name": string,
33
"creation_data": Date,
44
"dataset_id": string,
55
"questions": [Question]
66
}
77

8+
export interface DatasetDesc {
9+
"name": string,
10+
"creation_data": Date,
11+
"dataset_id": string,
12+
"nr_questions": number,
13+
"clusters_computed": number
14+
}
15+
816
export interface Question {
917
"question_id": string,
1018
"text": string

‎frontend/src/pages/tagging/DatasetSelection.tsx

Lines changed: 89 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
1-
import React, {useState} from 'react';
2-
import {Table, TableBody, TableContainer, TableHead, TableRow, Paper, Button} from '@material-ui/core';
1+
import React, {useEffect, useState} from 'react';
2+
import {Table, TableBody, TableContainer, TableHead, TableRow, Paper, Button, LinearProgress} from '@material-ui/core';
33
import {JSONLoader} from '../../helpers/LoaderHelper';
44
import {useHistory} from 'react-router-dom'
55
import {StyledTableRow, StyledTableCell, useStyles} from "../../components/styled/StyledTable";
66
import {downloadDatasetHelper} from "../../helpers/DownloadHelper";
77
import {Assignment, AssignmentLate, CloudDownload} from "@material-ui/icons";
8+
import {DatasetDesc} from "../../interfaces/Dataset";
89

910

1011
const {TAGGING_SERVICE_URL} = require('../../../config.json')
@@ -30,8 +31,8 @@ function requestUserId() {
3031
function DatasetSelection() {
3132
const router = useHistory() // TODO: use router from next once integrated, fix import
3233

33-
const [datasets, setDatasets] = useState([])
34-
const [loaded, setLoaded] = useState(false)
34+
const [datasets, setDatasets] = useState<DatasetDesc[]>([])
35+
const [loaded, setLoaded] = useState<boolean>(false)
3536

3637
const classes = useStyles();
3738
const url = TAGGING_SERVICE_URL + "/datasets/list"
@@ -43,6 +44,14 @@ function DatasetSelection() {
4344
setLoaded(true)
4445
}
4546

47+
useEffect(() => {
48+
const interval = setInterval(() => setLoaded(false), 5000);
49+
return () => {
50+
clearInterval(interval);
51+
};
52+
}, []);
53+
54+
4655
return (
4756
<TableContainer component={Paper} style={{
4857
width: '98%',
@@ -57,45 +66,82 @@ function DatasetSelection() {
5766
</TableRow>
5867
</TableHead>
5968
<TableBody>
60-
{datasets.map((row: { id: string, name: string, date: string }) => (
61-
<StyledTableRow key={row.id}>
62-
<StyledTableCell component="th" scope="row" onClick={
63-
() => redirect(row.id, "/taggingUI/tagView/", router)
64-
}>
65-
{row.name}
66-
</StyledTableCell>
67-
<StyledTableCell align="right" onClick={
68-
() => redirect(row.id, "/taggingUI/tagView/", router)
69-
}>{row.date}</StyledTableCell>
70-
<StyledTableCell align={"right"}>
71-
<Button
72-
title={"Download"}
73-
variant="outlined" color="primary"
74-
href="#outlined-buttons"
75-
onClick={() => downloadDatasetHelper(row.id, row.name)}>
76-
<CloudDownload/>
77-
</Button>
78-
<Button
79-
title={"Summary"}
80-
variant="outlined"
81-
color="primary"
82-
onClick={
83-
() => redirect(row.id, "/taggingUI/summary/", router)
84-
}>
85-
<Assignment/>
86-
</Button>
87-
<Button
88-
title={"Diff Data"}
89-
variant="outlined"
90-
color="primary"
91-
onClick={
92-
() => redirect(row.id, "/taggingUI/mergeView/", router)
93-
}>
94-
<AssignmentLate/>
95-
</Button>
96-
</StyledTableCell>
97-
</StyledTableRow>
98-
))}
69+
{datasets.map((dataset: DatasetDesc) => {
70+
const loading_cluster = dataset.clusters_computed != dataset.nr_questions
71+
// const needed_time_s = 1000 * 60 * 2 * dataset.nr_questions
72+
// const started = new Date(dataset.creation_data)
73+
// const now = new Date()
74+
//
75+
// const time_left = needed_time_s - (now.getTime() - started.getTime()) - now.getTimezoneOffset() * 1000 * 60
76+
//
77+
// const time_left_minutes = Math.floor(time_left / (60 * 1000))
78+
// const seconds = Math.floor((time_left - time_left_minutes * (60 * 1000)) / (1000))
79+
//
80+
// const time_left_seconds = seconds < 10 ? '0' + seconds : seconds
81+
82+
if (dataset.clusters_computed != dataset.nr_questions) {
83+
return (
84+
<StyledTableRow key={dataset.dataset_id}>
85+
<StyledTableCell component={'th'} scope={'row'}>
86+
{dataset.name}
87+
</StyledTableCell>
88+
<StyledTableCell component={'th'} scope={'row'}>
89+
<LinearProgress
90+
variant={'determinate'}
91+
value={Math.ceil(100 * dataset.clusters_computed / dataset.nr_questions)}
92+
/>
93+
</StyledTableCell>
94+
<StyledTableCell component={'th'} scope={'row'} style={{textAlign: 'end'}}>
95+
{`${dataset.clusters_computed}/${dataset.nr_questions}`}
96+
</StyledTableCell>
97+
</StyledTableRow>
98+
)
99+
}
100+
101+
return (
102+
<StyledTableRow key={dataset.dataset_id}>
103+
<StyledTableCell component="th" scope="row" onClick={
104+
() => redirect(dataset.dataset_id, "/taggingUI/tagView/", router)}>
105+
{dataset.name}
106+
</StyledTableCell>
107+
<StyledTableCell align="right" onClick={
108+
() => redirect(dataset.dataset_id, "/taggingUI/tagView/", router)
109+
}>
110+
{dataset.creation_data}
111+
</StyledTableCell>
112+
<StyledTableCell align={"right"}>
113+
<Button
114+
title={"Download"}
115+
variant="outlined" color="primary"
116+
href="#outlined-buttons"
117+
disabled={loading_cluster}
118+
onClick={() => downloadDatasetHelper(dataset.dataset_id, dataset.name)}>
119+
<CloudDownload/>
120+
</Button>
121+
<Button
122+
title={"Summary"}
123+
variant="outlined"
124+
disabled={loading_cluster}
125+
color="primary"
126+
onClick={
127+
() => redirect(dataset.dataset_id, "/taggingUI/summary/", router)
128+
}>
129+
<Assignment/>
130+
</Button>
131+
<Button
132+
title={"Diff Data"}
133+
variant="outlined"
134+
disabled={loading_cluster}
135+
color="primary"
136+
onClick={
137+
() => redirect(dataset.dataset_id, "/taggingUI/mergeView/", router)
138+
}>
139+
<AssignmentLate/>
140+
</Button>
141+
</StyledTableCell>
142+
</StyledTableRow>
143+
)
144+
})}
99145
</TableBody>
100146
</Table>
101147
</TableContainer>

‎tagging-service/.dockerignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
test
2+
cache
23
*.iml

‎tagging-service/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,4 @@ COPY . /tagging_service
88

99
RUN pip install -r requirements.txt
1010

11-
CMD flask run --host=0.0.0.0
11+
CMD gunicorn --preload --workers $NR_WORKERS --reload --timeout 90 -b 0.0.0.0:5000 flaskr:my_app

‎tagging-service/flaskr/__init__.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,3 +28,5 @@ def handle_invalid_usage(error):
2828
if __name__ == "__main__":
2929
app = create_app()
3030
app.run(debug=True, host='0.0.0.0', threaded=True)
31+
32+
my_app = create_app()

‎tagging-service/flaskr/config/__init__.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,9 @@
44
# https://flask-caching.readthedocs.io/en/latest/
55
config = {
66
"DEBUG": True, # some Flask specific configs
7-
"CACHE_TYPE": "simple", # Flask-Caching related configs
7+
"CACHE_TYPE": "filesystem", # Flask-Caching related configs
88
"CACHE_DIR": "cache",
9+
"CACHE_THRESHOLD": 1000,
910
"CACHE_DEFAULT_TIMEOUT": 0,
1011
}
1112
cache = Cache(config=config)

‎tagging-service/flaskr/endpoints/__init__.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
from flaskr.endpoints.upload_api import api as ns1
33
from flaskr.endpoints.progmiscon_api import api as ns2
44
from flaskr.endpoints.datasets_api import api as ns3
5+
from flaskr.endpoints.cluster_api import api as ns4
56

67
api = Api(version='1.0',
78
title='tagging Service',
@@ -12,3 +13,4 @@
1213
api.add_namespace(ns1)
1314
api.add_namespace(ns2)
1415
api.add_namespace(ns3)
16+
api.add_namespace(ns4)

‎tagging-service/flaskr/endpoints/cluster_api.py

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
from flask import request
2+
from flask_restx import Namespace, Resource, fields
3+
from flaskr.endpoints.upload_api import ANSWER
4+
from flaskr import cache
5+
from flaskr.util.mongo_helper import get_cluster, save_cluster
6+
7+
api = Namespace('clusters', description='API to view available datasets')
8+
9+
10+
CLUSTERED_ANSWERS = api.model('Clustered Answers', {
11+
'dataset_id': fields.String(required=True, readonly=True, description='ID of the dataset',
12+
example='603501f39175ac3898e094cc'),
13+
'question_id': fields.String(required=True, readonly=True, description='ID of the question',
14+
example='6035089963cf6ef09a9c418e'),
15+
'clusters': fields.List(fields.List(fields.Nested(ANSWER)))
16+
})
17+
18+
19+
@api.route('/dataset/<string:dataset_id>/question/<string:question_id>/user/<string:user_id>')
20+
@api.doc(description='API to get clusters for current user',
21+
params={
22+
'dataset_id': 'ID of the dataset',
23+
'user_id': 'ID of the user'
24+
})
25+
class Clusters(Resource):
26+
@api.marshal_list_with(CLUSTERED_ANSWERS)
27+
@cache.memoize()
28+
def get(self, dataset_id, question_id, user_id):
29+
return get_cluster(dataset_id=dataset_id, question_id=question_id, user_id=user_id)[0]
30+
31+
@api.expect(CLUSTERED_ANSWERS)
32+
def post(self, dataset_id, question_id, user_id):
33+
data = request.get_json()
34+
save_cluster(dataset_id=dataset_id, question_id=question_id, user_id=user_id,
35+
cluster=data)
36+
cache.delete_memoized(Clusters.get, dataset_id, question_id, user_id)

‎tagging-service/flaskr/endpoints/datasets_api.py

Lines changed: 0 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,6 @@
22
from flask_restx import Namespace, Resource, fields
33

44
import flaskr.util.mongo_helper as db
5-
from flaskr.endpoints.upload_api import ANSWER
6-
from flaskr import cache
75

86
api = Namespace('datasets', description='API to view available datasets')
97

@@ -40,13 +38,6 @@
4038
example='603501f39175ac3898e094cc')
4139
})
4240

43-
CLUSTERED_ANSWERS = api.model('Clustered Answers', {
44-
'dataset_id': fields.String(required=True, readonly=True, description='ID of the dataset',
45-
example='603501f39175ac3898e094cc'),
46-
'question_id': fields.String(required=True, readonly=True, description='ID of the question',
47-
example='6035089963cf6ef09a9c418e'),
48-
'clusters': fields.List(fields.List(fields.Nested(ANSWER)))
49-
})
5041

5142

5243
@api.route('/tagged-datasets')
@@ -124,20 +115,3 @@ def get(self, dataset_id, question_id):
124115
return db.get_tagged_question(dataset_id=dataset_id, question_id=question_id)
125116

126117

127-
@api.route('/clusters/dataset/<string:dataset_id>/question/<string:question_id>/user/<string:user_id>')
128-
@api.doc(description='API to get clusters for current user',
129-
params={
130-
'dataset_id': 'ID of the dataset',
131-
'user_id': 'ID of the user'
132-
})
133-
class Clusters(Resource):
134-
@api.marshal_list_with(CLUSTERED_ANSWERS)
135-
@cache.memoize()
136-
def get(self, dataset_id, question_id, user_id):
137-
return db.get_cluster(dataset_id=dataset_id, question_id=question_id, user_id=user_id)[0]
138-
139-
@api.expect(CLUSTERED_ANSWERS)
140-
def post(self, dataset_id, question_id, user_id):
141-
data = request.get_json()
142-
db.save_cluster(dataset_id=dataset_id, question_id=question_id, user_id=user_id, cluster=data)
143-
cache.delete_memoized(Clusters.get, dataset_id, user_id)

0 commit comments

Comments
 (0)