From 3524e9e82f328dfca1f9441a361700d56a051cba Mon Sep 17 00:00:00 2001 From: Daniel Cloud Date: Wed, 27 Jan 2021 16:51:51 -0500 Subject: [PATCH 01/64] Remove --runInBand --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 94b2ee5164..12f042baf1 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "server:debug": "nodemon src/index.js --exec babel-node --inspect", "client": "yarn --cwd frontend start", "test": "jest src", - "test:ci": "cross-env JEST_JUNIT_OUTPUT_DIR=reports JEST_JUNIT_OUTPUT_NAME=unit.xml POSTGRES_USERNAME=postgres POSTGRES_DB=ttasmarthub CURRENT_USER_ID=5 CI=true jest src --runInBand --coverage --reporters=default --reporters=jest-junit", + "test:ci": "cross-env JEST_JUNIT_OUTPUT_DIR=reports JEST_JUNIT_OUTPUT_NAME=unit.xml POSTGRES_USERNAME=postgres POSTGRES_DB=ttasmarthub CURRENT_USER_ID=5 CI=true jest src --coverage --reporters=default --reporters=jest-junit", "test:all": "yarn test:ci && yarn --cwd frontend test:ci", "lint": "eslint src", "lint:ci": "eslint -f eslint-formatter-multiple src", From 3c9b0b161f2de97a31163eeaafa2c797a2209bdc Mon Sep 17 00:00:00 2001 From: Chuck McAndrew Date: Thu, 28 Jan 2021 14:43:03 -0500 Subject: [PATCH 02/64] Start adding front end styling for file upload --- frontend/src/components/FileUploader.css | 14 ++++ frontend/src/components/FileUploader.js | 73 ++++++++++++------- frontend/src/components/Navigator/index.js | 4 +- frontend/src/fetchers/File.js | 9 +++ .../ActivityReport/Pages/topicsResources.js | 8 +- frontend/src/pages/ActivityReport/index.js | 1 + 6 files changed, 77 insertions(+), 32 deletions(-) create mode 100644 frontend/src/fetchers/File.js diff --git a/frontend/src/components/FileUploader.css b/frontend/src/components/FileUploader.css index d14ce5f7ea..cf0e827586 100644 --- a/frontend/src/components/FileUploader.css +++ b/frontend/src/components/FileUploader.css @@ -23,3 +23,17 @@ .fa-stack { width: 1em; } + +.files-table { + width: 100%; + border-collapse: collapse; + /* border: 1px solid #979797; */ +} + +/* .files-table--file-name { + width: 50%; +} */ + +.files-table td { + text-align: center; +} \ No newline at end of file diff --git a/frontend/src/components/FileUploader.js b/frontend/src/components/FileUploader.js index 6c68e5eb05..d4f78a6c8e 100644 --- a/frontend/src/components/FileUploader.js +++ b/frontend/src/components/FileUploader.js @@ -10,7 +10,7 @@ import PropTypes from 'prop-types'; import { useDropzone } from 'react-dropzone'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; -import { faTimes, faCircle } from '@fortawesome/free-solid-svg-icons'; +import { faTrash } from '@fortawesome/free-solid-svg-icons'; import { Tag, Button, Grid } from '@trussworks/react-uswds'; import './FileUploader.css'; @@ -18,6 +18,7 @@ import './FileUploader.css'; function Dropzone(props) { const { onChange } = props; const onDrop = (e) => { + console.log(e) onChange(e); }; const { getRootProps, getInputProps } = useDropzone({ onDrop }); @@ -37,30 +38,23 @@ function Dropzone(props) { }; const textStyle = { - textAlign: 'center', - fontSize: '16px', + textAlign: 'left', + fontSize: '1.25rem', + backgroundColor: "#0166AB", + color: "white", + padding: ".5rem", + borderRadius: "5px", }; - const linkStyle = { - cursor: 'pointer', - color: 'blue', - textDecoration: 'underline', - }; return (
-

- Drag and drop your files here - {' '} -
- or -
- Browse files -

+
); } @@ -81,13 +75,33 @@ const FileUploader = ({ onChange, files }) => { return ( <> - + + + + + + + + {files.map((file, index) => ( - - -
- {file.name} -
+ + + + + + + + ))} - + +
+ Name + + Size + + Status + +
+ {file.name} + + {`${(file.size / 1000).toFixed(1) } KB`} + + Uploaded + - - +
+ ); }; diff --git a/frontend/src/components/Navigator/index.js b/frontend/src/components/Navigator/index.js index 1b7f0c2650..9a5e59a53c 100644 --- a/frontend/src/components/Navigator/index.js +++ b/frontend/src/components/Navigator/index.js @@ -29,6 +29,7 @@ function Navigator({ additionalData, onSave, autoSaveInterval, + reportId, }) { const [formData, updateFormData] = useState(initialData); const [errorMessage, updateErrorMessage] = useState(); @@ -131,6 +132,7 @@ function Navigator({ submitted, onFormSubmit, additionalData, + reportId, )} {!page.review && ( @@ -142,7 +144,7 @@ function Navigator({ onSubmit={handleSubmit(onContinue)} className="smart-hub--form-large" > - {page.render(hookForm, additionalData)} + {page.render(hookForm, additionalData, reportId)} diff --git a/frontend/src/fetchers/File.js b/frontend/src/fetchers/File.js new file mode 100644 index 0000000000..31389dca62 --- /dev/null +++ b/frontend/src/fetchers/File.js @@ -0,0 +1,9 @@ +import join from 'url-join'; +import { get, put, post } from './index'; + +const activityReportUrl = join('/', 'api', 'files'); + +export const uploadFile = async (data) => { + const res = await post(activityReportUrl, data); + return res.json(); +}; \ No newline at end of file diff --git a/frontend/src/pages/ActivityReport/Pages/topicsResources.js b/frontend/src/pages/ActivityReport/Pages/topicsResources.js index 86bc9df499..11fb22fafb 100644 --- a/frontend/src/pages/ActivityReport/Pages/topicsResources.js +++ b/frontend/src/pages/ActivityReport/Pages/topicsResources.js @@ -15,6 +15,7 @@ import { topics } from '../constants'; const TopicsResources = ({ register, control, + reportId, }) => ( <> @@ -56,7 +57,7 @@ const TopicsResources = ({ defaultValue={[]} control={control} render={({ onChange, value }) => ( - + )} /> @@ -69,7 +70,7 @@ const TopicsResources = ({ defaultValue={[]} control={control} render={({ onChange, value }) => ( - + )} /> @@ -113,12 +114,13 @@ export default { path: 'topics-resources', sections, review: false, - render: (hookForm) => { + render: (hookForm, additionalData, reportId) => { const { control, register } = hookForm; return ( ); }, diff --git a/frontend/src/pages/ActivityReport/index.js b/frontend/src/pages/ActivityReport/index.js index 63d73e048d..e37542bc44 100644 --- a/frontend/src/pages/ActivityReport/index.js +++ b/frontend/src/pages/ActivityReport/index.js @@ -137,6 +137,7 @@ function ActivityReport({ match }) {

New activity report for Region 14

Date: Thu, 28 Jan 2021 15:14:53 -0500 Subject: [PATCH 03/64] Style file upload --- frontend/src/components/FileUploader.css | 14 +++++-- frontend/src/components/FileUploader.js | 48 +++++++++++++++--------- 2 files changed, 41 insertions(+), 21 deletions(-) diff --git a/frontend/src/components/FileUploader.css b/frontend/src/components/FileUploader.css index cf0e827586..90d6710490 100644 --- a/frontend/src/components/FileUploader.css +++ b/frontend/src/components/FileUploader.css @@ -30,10 +30,16 @@ /* border: 1px solid #979797; */ } -/* .files-table--file-name { - width: 50%; -} */ +.files-table--thead th, .files-table td { + padding: .5rem; + text-align: left; + font-size: .9rem; +} +.files-table--container { + border: solid 1px #979797; + min-height: 8rem; +} -.files-table td { +.files-table--empty { text-align: center; } \ No newline at end of file diff --git a/frontend/src/components/FileUploader.js b/frontend/src/components/FileUploader.js index d4f78a6c8e..b5b78b199e 100644 --- a/frontend/src/components/FileUploader.js +++ b/frontend/src/components/FileUploader.js @@ -63,30 +63,27 @@ Dropzone.propTypes = { onChange: PropTypes.func.isRequired, }; -const FileUploader = ({ onChange, files }) => { - const onFilesAdded = (newFiles) => { - onChange([...files, ...newFiles]); - }; - - const onFileRemoved = (removedFileIndex) => { - onChange(files.filter((f, index) => (index !== removedFileIndex))); - }; - +const FileTable = ({onFileRemoved, files}) => { + let msg + if (files.length === 0) { + msg = ( +

No files uploaded

+ ) + } return ( - <> - - - -
+
+ + + - - - @@ -120,6 +117,23 @@ const FileUploader = ({ onChange, files }) => { ))}
Name + Size + Status +
+ { msg } +
+ ); +} +const FileUploader = ({ onChange, files }) => { + const onFilesAdded = (newFiles) => { + onChange([...files, ...newFiles]); + }; + + const onFileRemoved = (removedFileIndex) => { + onChange(files.filter((f, index) => (index !== removedFileIndex))); + }; + + return ( + <> + + ); From 9d688011a5a88582183eee82380412abfe77a93d Mon Sep 17 00:00:00 2001 From: Chuck McAndrew Date: Thu, 28 Jan 2021 16:26:31 -0500 Subject: [PATCH 04/64] enable file upload --- frontend/src/components/FileUploader.js | 27 ++++++++++++++++--- frontend/src/fetchers/File.js | 13 ++++++--- .../ActivityReport/Pages/topicsResources.js | 4 +-- 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/frontend/src/components/FileUploader.js b/frontend/src/components/FileUploader.js index b5b78b199e..3582f60de9 100644 --- a/frontend/src/components/FileUploader.js +++ b/frontend/src/components/FileUploader.js @@ -8,16 +8,37 @@ import React from 'react'; import PropTypes from 'prop-types'; import { useDropzone } from 'react-dropzone'; +import * as FS from 'fs'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { faTrash } from '@fortawesome/free-solid-svg-icons'; import { Tag, Button, Grid } from '@trussworks/react-uswds'; +import { uploadFile } from '../fetchers/File'; import './FileUploader.css'; function Dropzone(props) { - const { onChange } = props; + const { onChange, id, reportId } = props; const onDrop = (e) => { + let attachmentType; + if (id === 'attachments') { + attachmentType = 'ATTACHMENT'; + } else if (id === "other-resources") { + attachmentType = 'RESOURCE'; + } + e.forEach( async (file) => { + try { + const data = new FormData() + data.append("reportId", reportId) + data.append("attachmentType", attachmentType) + data.append("file", file) + await uploadFile(data) + } catch (error) { + console.log(error) + } + + }); + console.log(e) onChange(e); }; @@ -121,7 +142,7 @@ const FileTable = ({onFileRemoved, files}) => { ); } -const FileUploader = ({ onChange, files }) => { +const FileUploader = ({ onChange, files, reportId, id }) => { const onFilesAdded = (newFiles) => { onChange([...files, ...newFiles]); }; @@ -132,7 +153,7 @@ const FileUploader = ({ onChange, files }) => { return ( <> - + diff --git a/frontend/src/fetchers/File.js b/frontend/src/fetchers/File.js index 31389dca62..275d96f6c0 100644 --- a/frontend/src/fetchers/File.js +++ b/frontend/src/fetchers/File.js @@ -1,9 +1,16 @@ import join from 'url-join'; -import { get, put, post } from './index'; +import { get } from './index'; const activityReportUrl = join('/', 'api', 'files'); export const uploadFile = async (data) => { - const res = await post(activityReportUrl, data); - return res.json(); + const res = await fetch(activityReportUrl, { + method: 'POST', + credentials: 'same-origin', + body: data + }); + if (!res.ok) { + throw new Error(res.statusText); + } + return res; }; \ No newline at end of file diff --git a/frontend/src/pages/ActivityReport/Pages/topicsResources.js b/frontend/src/pages/ActivityReport/Pages/topicsResources.js index 11fb22fafb..2bc4cb9293 100644 --- a/frontend/src/pages/ActivityReport/Pages/topicsResources.js +++ b/frontend/src/pages/ActivityReport/Pages/topicsResources.js @@ -57,7 +57,7 @@ const TopicsResources = ({ defaultValue={[]} control={control} render={({ onChange, value }) => ( - + )} /> @@ -70,7 +70,7 @@ const TopicsResources = ({ defaultValue={[]} control={control} render={({ onChange, value }) => ( - + )} /> From 1932a1b3dc3679522bbc2a285dcc4ed6e5b65f5d Mon Sep 17 00:00:00 2001 From: Chuck McAndrew Date: Thu, 28 Jan 2021 14:43:03 -0500 Subject: [PATCH 05/64] Start adding front end styling for file upload --- frontend/src/components/FileUploader.css | 14 ++++ frontend/src/components/FileUploader.js | 73 ++++++++++++------- frontend/src/components/Navigator/index.js | 4 +- frontend/src/fetchers/File.js | 9 +++ .../ActivityReport/Pages/topicsResources.js | 8 +- frontend/src/pages/ActivityReport/index.js | 1 + 6 files changed, 77 insertions(+), 32 deletions(-) create mode 100644 frontend/src/fetchers/File.js diff --git a/frontend/src/components/FileUploader.css b/frontend/src/components/FileUploader.css index d14ce5f7ea..cf0e827586 100644 --- a/frontend/src/components/FileUploader.css +++ b/frontend/src/components/FileUploader.css @@ -23,3 +23,17 @@ .fa-stack { width: 1em; } + +.files-table { + width: 100%; + border-collapse: collapse; + /* border: 1px solid #979797; */ +} + +/* .files-table--file-name { + width: 50%; +} */ + +.files-table td { + text-align: center; +} \ No newline at end of file diff --git a/frontend/src/components/FileUploader.js b/frontend/src/components/FileUploader.js index 6c68e5eb05..d4f78a6c8e 100644 --- a/frontend/src/components/FileUploader.js +++ b/frontend/src/components/FileUploader.js @@ -10,7 +10,7 @@ import PropTypes from 'prop-types'; import { useDropzone } from 'react-dropzone'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; -import { faTimes, faCircle } from '@fortawesome/free-solid-svg-icons'; +import { faTrash } from '@fortawesome/free-solid-svg-icons'; import { Tag, Button, Grid } from '@trussworks/react-uswds'; import './FileUploader.css'; @@ -18,6 +18,7 @@ import './FileUploader.css'; function Dropzone(props) { const { onChange } = props; const onDrop = (e) => { + console.log(e) onChange(e); }; const { getRootProps, getInputProps } = useDropzone({ onDrop }); @@ -37,30 +38,23 @@ function Dropzone(props) { }; const textStyle = { - textAlign: 'center', - fontSize: '16px', + textAlign: 'left', + fontSize: '1.25rem', + backgroundColor: "#0166AB", + color: "white", + padding: ".5rem", + borderRadius: "5px", }; - const linkStyle = { - cursor: 'pointer', - color: 'blue', - textDecoration: 'underline', - }; return (
-

- Drag and drop your files here - {' '} -
- or -
- Browse files -

+
); } @@ -81,13 +75,33 @@ const FileUploader = ({ onChange, files }) => { return ( <> - + + + + + + + + {files.map((file, index) => ( - - -
- {file.name} -
+ + + + + + + + ))} - + +
+ Name + + Size + + Status + +
+ {file.name} + + {`${(file.size / 1000).toFixed(1) } KB`} + + Uploaded + - - +
+ ); }; diff --git a/frontend/src/components/Navigator/index.js b/frontend/src/components/Navigator/index.js index 1b7f0c2650..9a5e59a53c 100644 --- a/frontend/src/components/Navigator/index.js +++ b/frontend/src/components/Navigator/index.js @@ -29,6 +29,7 @@ function Navigator({ additionalData, onSave, autoSaveInterval, + reportId, }) { const [formData, updateFormData] = useState(initialData); const [errorMessage, updateErrorMessage] = useState(); @@ -131,6 +132,7 @@ function Navigator({ submitted, onFormSubmit, additionalData, + reportId, )} {!page.review && ( @@ -142,7 +144,7 @@ function Navigator({ onSubmit={handleSubmit(onContinue)} className="smart-hub--form-large" > - {page.render(hookForm, additionalData)} + {page.render(hookForm, additionalData, reportId)} diff --git a/frontend/src/fetchers/File.js b/frontend/src/fetchers/File.js new file mode 100644 index 0000000000..31389dca62 --- /dev/null +++ b/frontend/src/fetchers/File.js @@ -0,0 +1,9 @@ +import join from 'url-join'; +import { get, put, post } from './index'; + +const activityReportUrl = join('/', 'api', 'files'); + +export const uploadFile = async (data) => { + const res = await post(activityReportUrl, data); + return res.json(); +}; \ No newline at end of file diff --git a/frontend/src/pages/ActivityReport/Pages/topicsResources.js b/frontend/src/pages/ActivityReport/Pages/topicsResources.js index 86bc9df499..11fb22fafb 100644 --- a/frontend/src/pages/ActivityReport/Pages/topicsResources.js +++ b/frontend/src/pages/ActivityReport/Pages/topicsResources.js @@ -15,6 +15,7 @@ import { topics } from '../constants'; const TopicsResources = ({ register, control, + reportId, }) => ( <> @@ -56,7 +57,7 @@ const TopicsResources = ({ defaultValue={[]} control={control} render={({ onChange, value }) => ( - + )} /> @@ -69,7 +70,7 @@ const TopicsResources = ({ defaultValue={[]} control={control} render={({ onChange, value }) => ( - + )} /> @@ -113,12 +114,13 @@ export default { path: 'topics-resources', sections, review: false, - render: (hookForm) => { + render: (hookForm, additionalData, reportId) => { const { control, register } = hookForm; return ( ); }, diff --git a/frontend/src/pages/ActivityReport/index.js b/frontend/src/pages/ActivityReport/index.js index 28f471ad3d..a498320eaa 100644 --- a/frontend/src/pages/ActivityReport/index.js +++ b/frontend/src/pages/ActivityReport/index.js @@ -153,6 +153,7 @@ function ActivityReport({ match }) {

New activity report for Region 14

Date: Thu, 28 Jan 2021 15:14:53 -0500 Subject: [PATCH 06/64] Style file upload --- frontend/src/components/FileUploader.css | 14 +++++-- frontend/src/components/FileUploader.js | 48 +++++++++++++++--------- 2 files changed, 41 insertions(+), 21 deletions(-) diff --git a/frontend/src/components/FileUploader.css b/frontend/src/components/FileUploader.css index cf0e827586..90d6710490 100644 --- a/frontend/src/components/FileUploader.css +++ b/frontend/src/components/FileUploader.css @@ -30,10 +30,16 @@ /* border: 1px solid #979797; */ } -/* .files-table--file-name { - width: 50%; -} */ +.files-table--thead th, .files-table td { + padding: .5rem; + text-align: left; + font-size: .9rem; +} +.files-table--container { + border: solid 1px #979797; + min-height: 8rem; +} -.files-table td { +.files-table--empty { text-align: center; } \ No newline at end of file diff --git a/frontend/src/components/FileUploader.js b/frontend/src/components/FileUploader.js index d4f78a6c8e..b5b78b199e 100644 --- a/frontend/src/components/FileUploader.js +++ b/frontend/src/components/FileUploader.js @@ -63,30 +63,27 @@ Dropzone.propTypes = { onChange: PropTypes.func.isRequired, }; -const FileUploader = ({ onChange, files }) => { - const onFilesAdded = (newFiles) => { - onChange([...files, ...newFiles]); - }; - - const onFileRemoved = (removedFileIndex) => { - onChange(files.filter((f, index) => (index !== removedFileIndex))); - }; - +const FileTable = ({onFileRemoved, files}) => { + let msg + if (files.length === 0) { + msg = ( +

No files uploaded

+ ) + } return ( - <> - - - -
+
+ + + - - - @@ -120,6 +117,23 @@ const FileUploader = ({ onChange, files }) => { ))}
Name + Size + Status +
+ { msg } +
+ ); +} +const FileUploader = ({ onChange, files }) => { + const onFilesAdded = (newFiles) => { + onChange([...files, ...newFiles]); + }; + + const onFileRemoved = (removedFileIndex) => { + onChange(files.filter((f, index) => (index !== removedFileIndex))); + }; + + return ( + <> + + ); From c7a7af21fa8db5f220aaae3140078719a8275297 Mon Sep 17 00:00:00 2001 From: Chuck McAndrew Date: Thu, 28 Jan 2021 16:26:31 -0500 Subject: [PATCH 07/64] enable file upload --- frontend/src/components/FileUploader.js | 27 ++++++++++++++++--- frontend/src/fetchers/File.js | 13 ++++++--- .../ActivityReport/Pages/topicsResources.js | 4 +-- 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/frontend/src/components/FileUploader.js b/frontend/src/components/FileUploader.js index b5b78b199e..3582f60de9 100644 --- a/frontend/src/components/FileUploader.js +++ b/frontend/src/components/FileUploader.js @@ -8,16 +8,37 @@ import React from 'react'; import PropTypes from 'prop-types'; import { useDropzone } from 'react-dropzone'; +import * as FS from 'fs'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { faTrash } from '@fortawesome/free-solid-svg-icons'; import { Tag, Button, Grid } from '@trussworks/react-uswds'; +import { uploadFile } from '../fetchers/File'; import './FileUploader.css'; function Dropzone(props) { - const { onChange } = props; + const { onChange, id, reportId } = props; const onDrop = (e) => { + let attachmentType; + if (id === 'attachments') { + attachmentType = 'ATTACHMENT'; + } else if (id === "other-resources") { + attachmentType = 'RESOURCE'; + } + e.forEach( async (file) => { + try { + const data = new FormData() + data.append("reportId", reportId) + data.append("attachmentType", attachmentType) + data.append("file", file) + await uploadFile(data) + } catch (error) { + console.log(error) + } + + }); + console.log(e) onChange(e); }; @@ -121,7 +142,7 @@ const FileTable = ({onFileRemoved, files}) => { ); } -const FileUploader = ({ onChange, files }) => { +const FileUploader = ({ onChange, files, reportId, id }) => { const onFilesAdded = (newFiles) => { onChange([...files, ...newFiles]); }; @@ -132,7 +153,7 @@ const FileUploader = ({ onChange, files }) => { return ( <> - + diff --git a/frontend/src/fetchers/File.js b/frontend/src/fetchers/File.js index 31389dca62..275d96f6c0 100644 --- a/frontend/src/fetchers/File.js +++ b/frontend/src/fetchers/File.js @@ -1,9 +1,16 @@ import join from 'url-join'; -import { get, put, post } from './index'; +import { get } from './index'; const activityReportUrl = join('/', 'api', 'files'); export const uploadFile = async (data) => { - const res = await post(activityReportUrl, data); - return res.json(); + const res = await fetch(activityReportUrl, { + method: 'POST', + credentials: 'same-origin', + body: data + }); + if (!res.ok) { + throw new Error(res.statusText); + } + return res; }; \ No newline at end of file diff --git a/frontend/src/pages/ActivityReport/Pages/topicsResources.js b/frontend/src/pages/ActivityReport/Pages/topicsResources.js index 11fb22fafb..2bc4cb9293 100644 --- a/frontend/src/pages/ActivityReport/Pages/topicsResources.js +++ b/frontend/src/pages/ActivityReport/Pages/topicsResources.js @@ -57,7 +57,7 @@ const TopicsResources = ({ defaultValue={[]} control={control} render={({ onChange, value }) => ( - + )} /> @@ -70,7 +70,7 @@ const TopicsResources = ({ defaultValue={[]} control={control} render={({ onChange, value }) => ( - + )} /> From 84a97a6bb2498ef1cfc59d6475f8aacc8ccff7e5 Mon Sep 17 00:00:00 2001 From: Chuck McAndrew Date: Thu, 28 Jan 2021 16:52:33 -0500 Subject: [PATCH 08/64] fmt --- frontend/src/components/FileUploader.js | 84 +++++++++---------------- 1 file changed, 29 insertions(+), 55 deletions(-) diff --git a/frontend/src/components/FileUploader.js b/frontend/src/components/FileUploader.js index 3582f60de9..bb27f886bb 100644 --- a/frontend/src/components/FileUploader.js +++ b/frontend/src/components/FileUploader.js @@ -8,11 +8,9 @@ import React from 'react'; import PropTypes from 'prop-types'; import { useDropzone } from 'react-dropzone'; -import * as FS from 'fs'; - import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { faTrash } from '@fortawesome/free-solid-svg-icons'; -import { Tag, Button, Grid } from '@trussworks/react-uswds'; +import { Button } from '@trussworks/react-uswds'; import { uploadFile } from '../fetchers/File'; import './FileUploader.css'; @@ -23,57 +21,31 @@ function Dropzone(props) { let attachmentType; if (id === 'attachments') { attachmentType = 'ATTACHMENT'; - } else if (id === "other-resources") { + } else if (id === 'other-resources') { attachmentType = 'RESOURCE'; } e.forEach( async (file) => { - try { - const data = new FormData() - data.append("reportId", reportId) - data.append("attachmentType", attachmentType) - data.append("file", file) - await uploadFile(data) - } catch (error) { - console.log(error) - } - + try { + const data = new FormData(); + data.append('reportId', reportId); + data.append('attachmentType', attachmentType); + data.append('file', file) + await uploadFile(data) + } catch (error) { + // eslint-disable-next-line no-console + console.log(error); + } }); - - console.log(e) onChange(e); }; const { getRootProps, getInputProps } = useDropzone({ onDrop }); - // I tried moving these styles to a css file and applying a class to the container - // and span. The styles were not being applied, it seems like the Dropzone library - // is messing with the styles somewhere - const containerStyle = { - maxWidth: '21rem', - height: '8rem', - display: 'flex', - justifyContent: 'center', - alignItems: 'center', - borderStyle: 'dashed', - borderWidth: '0.125rem', - borderColor: '#979797', - }; - - const textStyle = { - textAlign: 'left', - fontSize: '1.25rem', - backgroundColor: "#0166AB", - color: "white", - padding: ".5rem", - borderRadius: "5px", - }; - - return (
-
@@ -82,35 +54,37 @@ function Dropzone(props) { Dropzone.propTypes = { onChange: PropTypes.func.isRequired, + reportId: PropTypes.any.isRequired, + id: PropTypes.string.isRequired, }; const FileTable = ({onFileRemoved, files}) => { let msg if (files.length === 0) { msg = ( -

No files uploaded

+

No files uploaded

) } return ( -
- - -
+
+ + + - - - {files.map((file, index) => ( - From 9a3f9f502ebc12a157da68cf1fa710cc27f77c9f Mon Sep 17 00:00:00 2001 From: Chuck McAndrew Date: Thu, 28 Jan 2021 17:25:17 -0500 Subject: [PATCH 09/64] fmt --- frontend/src/components/FileUploader.js | 118 ++++++++++-------- frontend/src/components/Navigator/index.js | 1 + frontend/src/fetchers/File.js | 7 +- .../ActivityReport/Pages/topicsResources.js | 1 + 4 files changed, 69 insertions(+), 58 deletions(-) diff --git a/frontend/src/components/FileUploader.js b/frontend/src/components/FileUploader.js index bb27f886bb..8c7deab3cf 100644 --- a/frontend/src/components/FileUploader.js +++ b/frontend/src/components/FileUploader.js @@ -10,8 +10,8 @@ import PropTypes from 'prop-types'; import { useDropzone } from 'react-dropzone'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { faTrash } from '@fortawesome/free-solid-svg-icons'; -import { Button } from '@trussworks/react-uswds'; -import { uploadFile } from '../fetchers/File'; +import { Button } from '@trussworks/react-uswds'; +import uploadFile from '../fetchers/File'; import './FileUploader.css'; @@ -24,13 +24,13 @@ function Dropzone(props) { } else if (id === 'other-resources') { attachmentType = 'RESOURCE'; } - e.forEach( async (file) => { + e.forEach(async (file) => { try { const data = new FormData(); data.append('reportId', reportId); data.append('attachmentType', attachmentType); - data.append('file', file) - await uploadFile(data) + data.append('file', file); + await uploadFile(data); } catch (error) { // eslint-disable-next-line no-console console.log(error); @@ -45,7 +45,7 @@ function Dropzone(props) { {...getRootProps()} > - @@ -54,69 +54,77 @@ function Dropzone(props) { Dropzone.propTypes = { onChange: PropTypes.func.isRequired, - reportId: PropTypes.any.isRequired, + reportId: PropTypes.node.isRequired, id: PropTypes.string.isRequired, }; -const FileTable = ({onFileRemoved, files}) => { - let msg +const FileTable = ({ onFileRemoved, files }) => { + let msg; if (files.length === 0) { msg = ( -

No files uploaded

- ) +

No files uploaded

+ ); } return ( -
-
Name + Size + Status +
+ {file.name} @@ -121,14 +95,14 @@ const FileTable = ({onFileRemoved, files}) => {
- - - - - +
+
- Name - - Size - - Status - -
+ + + + + - {files.map((file, index) => ( - - - - - + {files.map((file, index) => ( + + + + + - + - ))} + ))}
+ Name + + Size + + Status +
- {file.name} - - {`${(file.size / 1000).toFixed(1) } KB`} - - Uploaded - - -
+ {file.name} + + {`${(file.size / 1000).toFixed(1)} KB`} + + Uploaded + + +
{ msg }
); -} -const FileUploader = ({ onChange, files, reportId, id }) => { +}; +FileTable.propTypes = { + onFileRemoved: PropTypes.func.isRequired, + files: PropTypes.arrayOf(PropTypes.instanceOf(File)), +}; +FileTable.defaultProps = { + files: [], +}; +const FileUploader = ({ + onChange, files, reportId, id, +}) => { const onFilesAdded = (newFiles) => { onChange([...files, ...newFiles]); }; @@ -129,7 +137,7 @@ const FileUploader = ({ onChange, files, reportId, id }) => { <> - + ); }; @@ -137,6 +145,8 @@ const FileUploader = ({ onChange, files, reportId, id }) => { FileUploader.propTypes = { onChange: PropTypes.func.isRequired, files: PropTypes.arrayOf(PropTypes.instanceOf(File)), + reportId: PropTypes.node.isRequired, + id: PropTypes.string.isRequired, }; FileUploader.defaultProps = { diff --git a/frontend/src/components/Navigator/index.js b/frontend/src/components/Navigator/index.js index 9a5e59a53c..a0cf8a44b0 100644 --- a/frontend/src/components/Navigator/index.js +++ b/frontend/src/components/Navigator/index.js @@ -172,6 +172,7 @@ Navigator.propTypes = { currentPage: PropTypes.string.isRequired, autoSaveInterval: PropTypes.number, additionalData: PropTypes.shape({}), + reportId: PropTypes.node.isRequired, }; Navigator.defaultProps = { diff --git a/frontend/src/fetchers/File.js b/frontend/src/fetchers/File.js index 275d96f6c0..a1a2f176b4 100644 --- a/frontend/src/fetchers/File.js +++ b/frontend/src/fetchers/File.js @@ -1,16 +1,15 @@ import join from 'url-join'; -import { get } from './index'; const activityReportUrl = join('/', 'api', 'files'); -export const uploadFile = async (data) => { +export default async function uploadFile(data) { const res = await fetch(activityReportUrl, { method: 'POST', credentials: 'same-origin', - body: data + body: data, }); if (!res.ok) { throw new Error(res.statusText); } return res; -}; \ No newline at end of file +} diff --git a/frontend/src/pages/ActivityReport/Pages/topicsResources.js b/frontend/src/pages/ActivityReport/Pages/topicsResources.js index 2bc4cb9293..994aacccd1 100644 --- a/frontend/src/pages/ActivityReport/Pages/topicsResources.js +++ b/frontend/src/pages/ActivityReport/Pages/topicsResources.js @@ -81,6 +81,7 @@ TopicsResources.propTypes = { register: PropTypes.func.isRequired, // eslint-disable-next-line react/forbid-prop-types control: PropTypes.object.isRequired, + reportId: PropTypes.node.isRequired, }; const sections = [ From 8a83029734c685dac555ccee4c1dd4971c8c1281 Mon Sep 17 00:00:00 2001 From: Chuck McAndrew Date: Fri, 29 Jan 2021 09:27:29 -0500 Subject: [PATCH 10/64] fix tests --- frontend/src/components/FileUploader.js | 7 +++++-- frontend/src/components/__tests__/FileUploader.js | 11 ++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/frontend/src/components/FileUploader.js b/frontend/src/components/FileUploader.js index 8c7deab3cf..a6a027e66a 100644 --- a/frontend/src/components/FileUploader.js +++ b/frontend/src/components/FileUploader.js @@ -67,8 +67,9 @@ const FileTable = ({ onFileRemoved, files }) => { } return (
- +
+ @@ -79,10 +80,12 @@ const FileTable = ({ onFileRemoved, files }) => { Status {files.map((file, index) => ( - + diff --git a/frontend/src/components/__tests__/FileUploader.js b/frontend/src/components/__tests__/FileUploader.js index e9f3ca4fc6..7498fba101 100644 --- a/frontend/src/components/__tests__/FileUploader.js +++ b/frontend/src/components/__tests__/FileUploader.js @@ -34,7 +34,7 @@ describe('FileUploader', () => { it('onDrop adds calls the onChange method', async () => { const mockOnChange = jest.fn(); const data = mockData([file('file')]); - const ui = ; + const ui = ; const { container, rerender } = render(ui); const dropzone = container.querySelector('div'); @@ -45,16 +45,17 @@ describe('FileUploader', () => { }); it('files are properly displayed', () => { - render( {}} files={[file('fileOne'), file('fileTwo')]} />); + render( {}} files={[file('fileOne'), file('fileTwo')]} />); expect(screen.getByText('fileOne')).toBeVisible(); expect(screen.getByText('fileTwo')).toBeVisible(); }); it('files can be removed', () => { const mockOnChange = jest.fn(); - render(); - const fileOne = screen.getByText('fileOne'); - fireEvent.click(fileOne.nextSibling); + render(); + const fileTwo = screen.getByText('fileTwo'); + console.log(fileTwo.parentNode.lastChild.firstChild) + fireEvent.click(fileTwo.parentNode.lastChild.firstChild); expect(mockOnChange).toHaveBeenCalledWith([file('fileTwo')]); }); From 68ba4c053b14bef566fd9e9ef5ca906d3978e570 Mon Sep 17 00:00:00 2001 From: Chuck McAndrew Date: Fri, 29 Jan 2021 17:10:16 -0500 Subject: [PATCH 11/64] update so attached files display --- frontend/src/components/FileUploader.js | 12 +++++------ .../src/components/__tests__/FileUploader.js | 4 ++-- .../ActivityReport/Pages/topicsResources.js | 8 +++---- .../20210129210854-add-fileSize-to-files.js | 14 +++++++++++++ src/models/activityReport.js | 3 ++- src/models/file.js | 4 ++++ src/routes/files/handlers.js | 15 +++++++++++-- src/routes/files/handlers.test.js | 10 +++++++++ src/services/activityReports.js | 21 ++++++++++++++++++- 9 files changed, 75 insertions(+), 16 deletions(-) create mode 100644 src/migrations/20210129210854-add-fileSize-to-files.js diff --git a/frontend/src/components/FileUploader.js b/frontend/src/components/FileUploader.js index a6a027e66a..0cb3433e8a 100644 --- a/frontend/src/components/FileUploader.js +++ b/frontend/src/components/FileUploader.js @@ -21,7 +21,7 @@ function Dropzone(props) { let attachmentType; if (id === 'attachments') { attachmentType = 'ATTACHMENT'; - } else if (id === 'other-resources') { + } else if (id === 'otherResources') { attachmentType = 'RESOURCE'; } e.forEach(async (file) => { @@ -31,12 +31,12 @@ function Dropzone(props) { data.append('attachmentType', attachmentType); data.append('file', file); await uploadFile(data); + onChange([{originalFileName: file.name, fileSize: file.size, status: "Uploaded"}]) } catch (error) { // eslint-disable-next-line no-console console.log(error); } }); - onChange(e); }; const { getRootProps, getInputProps } = useDropzone({ onDrop }); @@ -87,13 +87,13 @@ const FileTable = ({ onFileRemoved, files }) => { {files.map((file, index) => (
Name + +
{file.name}
- {file.name} + {file.originalFileName} - {`${(file.size / 1000).toFixed(1)} KB`} + {`${(file.fileSize / 1000).toFixed(1)} KB`} - Uploaded + {file.status} ); } @@ -81,13 +75,33 @@ const FileUploader = ({ onChange, files }) => { return ( <> - + + + + + + + + {files.map((file, index) => ( - - -
- {file.name} -
+ + + + + + + + ))} - + +
+ Name + + Size + + Status + +
+ {file.name} + + {`${(file.size / 1000).toFixed(1) } KB`} + + Uploaded + - - +
+ ); }; diff --git a/frontend/src/components/Navigator/index.js b/frontend/src/components/Navigator/index.js index 1b7f0c2650..9a5e59a53c 100644 --- a/frontend/src/components/Navigator/index.js +++ b/frontend/src/components/Navigator/index.js @@ -29,6 +29,7 @@ function Navigator({ additionalData, onSave, autoSaveInterval, + reportId, }) { const [formData, updateFormData] = useState(initialData); const [errorMessage, updateErrorMessage] = useState(); @@ -131,6 +132,7 @@ function Navigator({ submitted, onFormSubmit, additionalData, + reportId, )} {!page.review && ( @@ -142,7 +144,7 @@ function Navigator({ onSubmit={handleSubmit(onContinue)} className="smart-hub--form-large" > - {page.render(hookForm, additionalData)} + {page.render(hookForm, additionalData, reportId)} diff --git a/frontend/src/fetchers/File.js b/frontend/src/fetchers/File.js new file mode 100644 index 0000000000..31389dca62 --- /dev/null +++ b/frontend/src/fetchers/File.js @@ -0,0 +1,9 @@ +import join from 'url-join'; +import { get, put, post } from './index'; + +const activityReportUrl = join('/', 'api', 'files'); + +export const uploadFile = async (data) => { + const res = await post(activityReportUrl, data); + return res.json(); +}; \ No newline at end of file diff --git a/frontend/src/pages/ActivityReport/Pages/topicsResources.js b/frontend/src/pages/ActivityReport/Pages/topicsResources.js index 86bc9df499..11fb22fafb 100644 --- a/frontend/src/pages/ActivityReport/Pages/topicsResources.js +++ b/frontend/src/pages/ActivityReport/Pages/topicsResources.js @@ -15,6 +15,7 @@ import { topics } from '../constants'; const TopicsResources = ({ register, control, + reportId, }) => ( <> @@ -56,7 +57,7 @@ const TopicsResources = ({ defaultValue={[]} control={control} render={({ onChange, value }) => ( - + )} /> @@ -69,7 +70,7 @@ const TopicsResources = ({ defaultValue={[]} control={control} render={({ onChange, value }) => ( - + )} /> @@ -113,12 +114,13 @@ export default { path: 'topics-resources', sections, review: false, - render: (hookForm) => { + render: (hookForm, additionalData, reportId) => { const { control, register } = hookForm; return ( ); }, diff --git a/frontend/src/pages/ActivityReport/index.js b/frontend/src/pages/ActivityReport/index.js index 7177a288c5..5e1157122d 100644 --- a/frontend/src/pages/ActivityReport/index.js +++ b/frontend/src/pages/ActivityReport/index.js @@ -152,6 +152,7 @@ function ActivityReport({ match }) {

New activity report for Region 14

Date: Thu, 28 Jan 2021 15:14:53 -0500 Subject: [PATCH 23/64] Style file upload --- frontend/src/components/FileUploader.css | 14 +++++-- frontend/src/components/FileUploader.js | 48 +++++++++++++++--------- 2 files changed, 41 insertions(+), 21 deletions(-) diff --git a/frontend/src/components/FileUploader.css b/frontend/src/components/FileUploader.css index cf0e827586..90d6710490 100644 --- a/frontend/src/components/FileUploader.css +++ b/frontend/src/components/FileUploader.css @@ -30,10 +30,16 @@ /* border: 1px solid #979797; */ } -/* .files-table--file-name { - width: 50%; -} */ +.files-table--thead th, .files-table td { + padding: .5rem; + text-align: left; + font-size: .9rem; +} +.files-table--container { + border: solid 1px #979797; + min-height: 8rem; +} -.files-table td { +.files-table--empty { text-align: center; } \ No newline at end of file diff --git a/frontend/src/components/FileUploader.js b/frontend/src/components/FileUploader.js index d4f78a6c8e..b5b78b199e 100644 --- a/frontend/src/components/FileUploader.js +++ b/frontend/src/components/FileUploader.js @@ -63,30 +63,27 @@ Dropzone.propTypes = { onChange: PropTypes.func.isRequired, }; -const FileUploader = ({ onChange, files }) => { - const onFilesAdded = (newFiles) => { - onChange([...files, ...newFiles]); - }; - - const onFileRemoved = (removedFileIndex) => { - onChange(files.filter((f, index) => (index !== removedFileIndex))); - }; - +const FileTable = ({onFileRemoved, files}) => { + let msg + if (files.length === 0) { + msg = ( +

No files uploaded

+ ) + } return ( - <> - - - -
+
+ + + - - - @@ -120,6 +117,23 @@ const FileUploader = ({ onChange, files }) => { ))}
Name + Size + Status +
+ { msg } +
+ ); +} +const FileUploader = ({ onChange, files }) => { + const onFilesAdded = (newFiles) => { + onChange([...files, ...newFiles]); + }; + + const onFileRemoved = (removedFileIndex) => { + onChange(files.filter((f, index) => (index !== removedFileIndex))); + }; + + return ( + <> + + ); From e09e2da057b7da5ef32443722a6307eb0a01845f Mon Sep 17 00:00:00 2001 From: Chuck McAndrew Date: Thu, 28 Jan 2021 16:26:31 -0500 Subject: [PATCH 24/64] enable file upload --- frontend/src/components/FileUploader.js | 27 ++++++++++++++++--- frontend/src/fetchers/File.js | 13 ++++++--- .../ActivityReport/Pages/topicsResources.js | 4 +-- 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/frontend/src/components/FileUploader.js b/frontend/src/components/FileUploader.js index b5b78b199e..3582f60de9 100644 --- a/frontend/src/components/FileUploader.js +++ b/frontend/src/components/FileUploader.js @@ -8,16 +8,37 @@ import React from 'react'; import PropTypes from 'prop-types'; import { useDropzone } from 'react-dropzone'; +import * as FS from 'fs'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { faTrash } from '@fortawesome/free-solid-svg-icons'; import { Tag, Button, Grid } from '@trussworks/react-uswds'; +import { uploadFile } from '../fetchers/File'; import './FileUploader.css'; function Dropzone(props) { - const { onChange } = props; + const { onChange, id, reportId } = props; const onDrop = (e) => { + let attachmentType; + if (id === 'attachments') { + attachmentType = 'ATTACHMENT'; + } else if (id === "other-resources") { + attachmentType = 'RESOURCE'; + } + e.forEach( async (file) => { + try { + const data = new FormData() + data.append("reportId", reportId) + data.append("attachmentType", attachmentType) + data.append("file", file) + await uploadFile(data) + } catch (error) { + console.log(error) + } + + }); + console.log(e) onChange(e); }; @@ -121,7 +142,7 @@ const FileTable = ({onFileRemoved, files}) => { ); } -const FileUploader = ({ onChange, files }) => { +const FileUploader = ({ onChange, files, reportId, id }) => { const onFilesAdded = (newFiles) => { onChange([...files, ...newFiles]); }; @@ -132,7 +153,7 @@ const FileUploader = ({ onChange, files }) => { return ( <> - + diff --git a/frontend/src/fetchers/File.js b/frontend/src/fetchers/File.js index 31389dca62..275d96f6c0 100644 --- a/frontend/src/fetchers/File.js +++ b/frontend/src/fetchers/File.js @@ -1,9 +1,16 @@ import join from 'url-join'; -import { get, put, post } from './index'; +import { get } from './index'; const activityReportUrl = join('/', 'api', 'files'); export const uploadFile = async (data) => { - const res = await post(activityReportUrl, data); - return res.json(); + const res = await fetch(activityReportUrl, { + method: 'POST', + credentials: 'same-origin', + body: data + }); + if (!res.ok) { + throw new Error(res.statusText); + } + return res; }; \ No newline at end of file diff --git a/frontend/src/pages/ActivityReport/Pages/topicsResources.js b/frontend/src/pages/ActivityReport/Pages/topicsResources.js index 11fb22fafb..2bc4cb9293 100644 --- a/frontend/src/pages/ActivityReport/Pages/topicsResources.js +++ b/frontend/src/pages/ActivityReport/Pages/topicsResources.js @@ -57,7 +57,7 @@ const TopicsResources = ({ defaultValue={[]} control={control} render={({ onChange, value }) => ( - + )} /> @@ -70,7 +70,7 @@ const TopicsResources = ({ defaultValue={[]} control={control} render={({ onChange, value }) => ( - + )} /> From deac317ffba5ec7b11a52f7a7ae41e0e2b922a3a Mon Sep 17 00:00:00 2001 From: Daniel Cloud Date: Tue, 26 Jan 2021 15:42:12 -0500 Subject: [PATCH 25/64] Rename docker-compose file used for dynamic security scans --- .circleci/config.yml | 12 ++++++------ docker-compose-test.yml => docker-compose-dss.yml | 0 2 files changed, 6 insertions(+), 6 deletions(-) rename docker-compose-test.yml => docker-compose-dss.yml (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index b9adf5ba70..7265c80d1f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -240,12 +240,12 @@ jobs: - run: name: Start up local server command: | # production style build (single BE server with static FE) - docker-compose -f docker-compose-test.yml run --rm server yarn install --production=false - docker-compose -f docker-compose-test.yml run --rm server yarn --cwd frontend install --production=false - docker-compose -f docker-compose-test.yml run --rm server yarn build - docker-compose -f docker-compose-test.yml run --rm server yarn --cwd frontend run build - docker-compose -f docker-compose-test.yml up -d - docker-compose -f docker-compose-test.yml exec server yarn db:migrate:ci + docker-compose -f docker-compose-dss.yml run --rm server yarn install --production=false + docker-compose -f docker-compose-dss.yml run --rm server yarn --cwd frontend install --production=false + docker-compose -f docker-compose-dss.yml run --rm server yarn build + docker-compose -f docker-compose-dss.yml run --rm server yarn --cwd frontend run build + docker-compose -f docker-compose-dss.yml up -d + docker-compose -f docker-compose-dss.yml exec server yarn db:migrate:ci - run: name: Pull OWASP ZAP docker image command: docker pull owasp/zap2docker-weekly diff --git a/docker-compose-test.yml b/docker-compose-dss.yml similarity index 100% rename from docker-compose-test.yml rename to docker-compose-dss.yml From f3f90222c78658ca0e6616ede22afc188d3568fd Mon Sep 17 00:00:00 2001 From: Daniel Cloud Date: Wed, 27 Jan 2021 16:11:08 -0500 Subject: [PATCH 26/64] Create bin/docker script, which will make command composition easier --- bin/docker | 195 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100755 bin/docker diff --git a/bin/docker b/bin/docker new file mode 100755 index 0000000000..1d304f39e1 --- /dev/null +++ b/bin/docker @@ -0,0 +1,195 @@ +#!/bin/bash + +hr="==========" + +sequelizecli="node_modules/.bin/sequelize" + +setup() { + # First, setup up frontend and backend, install dependencies + # yarn docker:deps + setup_backend + setup_frontend + + db_setup "" + + # Stop the db manually? + # db_stop +} + +setup_backend() { + echo "${hr} Setting up backend ${hr}" + docker-compose run --rm backend yarn install +} + +setup_frontend() { + echo "${hr} Setting up frontend ${hr}" + docker-compose run --rm frontend yarn install +} + +teardown() { + docker-compose down --volumes +} + +db_migrate() { + local -a options=(status undo) + local option="${1}" + # If 'undo' param provided undo + if [[ "undo" == "${option}" ]]; then + echo "${hr} Undoing db migrations ${hr}" + docker-compose run --rm backend "$sequelizecli" db:migrate:undo + exit 0 + fi + + if [[ "status" == "${option}" ]]; then + echo "${hr} Checking db migration status ${hr}" + docker-compose run --rm backend "$sequelizecli" db:migrate:status + exit 0 + fi + + # Create/update database tables + # yarn docker:db:migrate + echo "${hr} Migrating db ${hr}" + docker-compose run --rm backend "$sequelizecli" db:migrate +} + +db_seed() { + local option="${1}" + # If 'undo' param provided undo + if [[ "undo" == "${option}" ]]; then + echo "${hr} Unseeding db ${hr}" + docker-compose run --rm backend "$sequelizecli" db:seed:undo:all + exit 0 + fi + + # Populate db with data + # yarn docker:db:seed + echo "${hr} Seeding database ${hr}" + docker-compose run --rm backend yarn db:seed +} + +db_setup() { + db_migrate "" + db_seed "" +} + +db_stop() { + docker-compose stop db +} + +# Need reliable way to remove db volume and only db volume +# db_destroy() { +# db_stop +# docker-compose rm -v -- db +# docker volume prune +# } + +db() { + declare -a options=(setup migrate seed stop) + option="${1}" + for i in "${options[@]}"; do + if [ "${i}" == "${option}" ]; then + shift 2>/dev/null + "db_${option}" "$@" + exit 0 + fi + done + + # If no matching option, print options + echo "db expects one of the following: ${options[*]}" + exit 1 +} + +test_frontend() { + echo "${hr} Running frontend tests ${hr}" + docker-compose run --rm frontend yarn test:ci +} + +test_backend() { + echo "${hr} Running backend tests ${hr}" + docker-compose run --rm backend yarn test:ci +} + +# tests plural, because `test` is a builtin shell command +tests() { + declare -a options=(frontend backend) + option="${1}" + + # Check if user specified 'frontend' or 'backend' option + if [[ -n $option ]]; then + for i in "${options[@]}"; do + if [ "${i}" == "${option}" ]; then + "test_${option}" + exit 0 + fi + done + fi + + # Otherwise, run both + test_frontend + test_backend + +} + +lint_backend() { + echo "${hr} Linting backend ${hr}" + docker-compose run --rm backend yarn lint:ci +} + +lint_frontend() { + echo "${hr} Linting frontend ${hr}" + docker-compose run --rm frontend yarn lint:ci +} + +lint() { + local -a options=(frontend backend) + option="${1}" + + for i in "${options[@]}"; do + if [[ "${i}" == "$option" ]]; then + "lint_${option}" + exit 0 + fi + done + + lint_frontend + lint_backend +} + +start() { + docker-compose up +} + +stop() { + docker-compose down +} + +# ====== Main ===== + +# Pull first param as subcommand, default to 'usage' if no args provided +subcommand="${1-usage}"; +# Shift parameters, so $@a has remaining params +shift 2>/dev/null + +# If subcommand is empty string, print usage +if [ -z "$subcommand" ]; then + usage + exit 1 +fi + +usage() { + echo "Please provide a subcommand: ${commands[*]}" +} + +# Array of subcommands +declare -a commands=(usage setup teardown start stop db lint tests) + +# Loop over commands array, look for match with $subcommand. Execute fn if match +for i in "${commands[@]}"; do + if [[ "${i}" == "${subcommand}" ]]; then + "${subcommand}" "$@" + exit 0 + fi +done + +# If we didn't run a subcommand and exit already, print usage +usage From 44af291dd235220a8ec2842d6a5c3442d42954e8 Mon Sep 17 00:00:00 2001 From: Daniel Cloud Date: Wed, 27 Jan 2021 16:51:51 -0500 Subject: [PATCH 27/64] Remove --runInBand --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 94b2ee5164..12f042baf1 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "server:debug": "nodemon src/index.js --exec babel-node --inspect", "client": "yarn --cwd frontend start", "test": "jest src", - "test:ci": "cross-env JEST_JUNIT_OUTPUT_DIR=reports JEST_JUNIT_OUTPUT_NAME=unit.xml POSTGRES_USERNAME=postgres POSTGRES_DB=ttasmarthub CURRENT_USER_ID=5 CI=true jest src --runInBand --coverage --reporters=default --reporters=jest-junit", + "test:ci": "cross-env JEST_JUNIT_OUTPUT_DIR=reports JEST_JUNIT_OUTPUT_NAME=unit.xml POSTGRES_USERNAME=postgres POSTGRES_DB=ttasmarthub CURRENT_USER_ID=5 CI=true jest src --coverage --reporters=default --reporters=jest-junit", "test:all": "yarn test:ci && yarn --cwd frontend test:ci", "lint": "eslint src", "lint:ci": "eslint -f eslint-formatter-multiple src", From 9b422f35b4f750430fa98d18149739de01937550 Mon Sep 17 00:00:00 2001 From: Daniel Cloud Date: Wed, 27 Jan 2021 17:45:18 -0500 Subject: [PATCH 28/64] Add option parsing and rename docker script --- bin/{docker => docker-run} | 101 ++++++++++++++++++++++++++++--------- 1 file changed, 78 insertions(+), 23 deletions(-) rename bin/{docker => docker-run} (65%) diff --git a/bin/docker b/bin/docker-run similarity index 65% rename from bin/docker rename to bin/docker-run index 1d304f39e1..765a26575d 100755 --- a/bin/docker +++ b/bin/docker-run @@ -2,7 +2,23 @@ hr="==========" -sequelizecli="node_modules/.bin/sequelize" +declare sequelizecli="node_modules/.bin/sequelize" + +declare -a environments=("local" "test") +declare env="local" + +# Array of subcommands +declare -a subcommands=( + usage + setup + teardown + start + stop + db + lint + tests +) + setup() { # First, setup up frontend and backend, install dependencies @@ -163,33 +179,72 @@ stop() { docker-compose down } +usage() { + # shellcheck disable=SC2086 + cat <&2 +Run commands in Docker containers +Usage: $0 [-h] [-e env] +Subcommands: + ${subcommands[*]} +EOF +} + # ====== Main ===== -# Pull first param as subcommand, default to 'usage' if no args provided -subcommand="${1-usage}"; -# Shift parameters, so $@a has remaining params -shift 2>/dev/null +main() { + # Parse -options + while getopts ":e:h" opt; do + case $opt in + e) + env="${OPTARG-local}" + local env_match=0 + for e in "${environments[@]}"; do + if [[ "$e" == "$env" ]]; then + env_match=1; + break; + fi + done + if [[ $env_match -eq 0 ]]; then + echo "Invalid environment '${env}'"; + exit 1; + fi + ;; + h) + usage + exit + ;; + \?) + echo "Invalid option" >&2 + echo + ;; + esac + done -# If subcommand is empty string, print usage -if [ -z "$subcommand" ]; then - usage - exit 1 -fi + # After parsing options, shift so $@ has only params + shift $((OPTIND -1)) -usage() { - echo "Please provide a subcommand: ${commands[*]}" -} + # Pull first param as subcommand, default to 'usage' if no args provided + local subcmd="${1-usage}"; -# Array of subcommands -declare -a commands=(usage setup teardown start stop db lint tests) + # Shift parameters, so $@a has remaining params + shift 2>/dev/null -# Loop over commands array, look for match with $subcommand. Execute fn if match -for i in "${commands[@]}"; do - if [[ "${i}" == "${subcommand}" ]]; then - "${subcommand}" "$@" - exit 0 + # If subcommand is empty string, print usage + if [ -z "$subcmd" ]; then + usage + exit 1 fi -done -# If we didn't run a subcommand and exit already, print usage -usage + # Loop over subcommands, look for match with $subcommand. Execute fn if match + for i in "${subcommands[@]}"; do + if [[ "${i}" == "${subcmd}" ]]; then + "${subcmd}" "$@" + exit 0 + fi + done + + # If we didn't run a subcommand and exit already, print usage + usage +} + +main "${@-}" From 84b24b86c079b0b66d341372991a13d64ef93aa9 Mon Sep 17 00:00:00 2001 From: Daniel Cloud Date: Wed, 27 Jan 2021 18:55:25 -0500 Subject: [PATCH 29/64] Multiple docker-compose configs for multiple environments --- bin/docker-run | 61 +++++++++++++++++++++++++++++----------- docker-compose.local.yml | 12 ++++++++ docker-compose.test.yml | 7 +++++ docker-compose.yml | 6 ---- 4 files changed, 64 insertions(+), 22 deletions(-) create mode 100644 docker-compose.local.yml create mode 100644 docker-compose.test.yml diff --git a/bin/docker-run b/bin/docker-run index 765a26575d..533eeaa2a4 100755 --- a/bin/docker-run +++ b/bin/docker-run @@ -5,7 +5,8 @@ hr="==========" declare sequelizecli="node_modules/.bin/sequelize" declare -a environments=("local" "test") -declare env="local" + +declare -a conf # Array of subcommands declare -a subcommands=( @@ -34,16 +35,18 @@ setup() { setup_backend() { echo "${hr} Setting up backend ${hr}" - docker-compose run --rm backend yarn install + docker-compose "${conf[@]}" run --rm backend yarn install } setup_frontend() { echo "${hr} Setting up frontend ${hr}" - docker-compose run --rm frontend yarn install + docker-compose "${conf[@]}" run --rm frontend yarn install } teardown() { - docker-compose down --volumes + docker-compose \ + "${conf[@]}" \ + down --volumes } db_migrate() { @@ -52,20 +55,20 @@ db_migrate() { # If 'undo' param provided undo if [[ "undo" == "${option}" ]]; then echo "${hr} Undoing db migrations ${hr}" - docker-compose run --rm backend "$sequelizecli" db:migrate:undo + docker-compose "${conf[@]}" run --rm backend "$sequelizecli" db:migrate:undo exit 0 fi if [[ "status" == "${option}" ]]; then echo "${hr} Checking db migration status ${hr}" - docker-compose run --rm backend "$sequelizecli" db:migrate:status + docker-compose "${conf[@]}" run --rm backend "$sequelizecli" db:migrate:status exit 0 fi # Create/update database tables # yarn docker:db:migrate echo "${hr} Migrating db ${hr}" - docker-compose run --rm backend "$sequelizecli" db:migrate + docker-compose "${conf[@]}" run --rm backend "$sequelizecli" db:migrate } db_seed() { @@ -73,14 +76,14 @@ db_seed() { # If 'undo' param provided undo if [[ "undo" == "${option}" ]]; then echo "${hr} Unseeding db ${hr}" - docker-compose run --rm backend "$sequelizecli" db:seed:undo:all + docker-compose "${conf[@]}" run --rm backend "$sequelizecli" db:seed:undo:all exit 0 fi # Populate db with data # yarn docker:db:seed echo "${hr} Seeding database ${hr}" - docker-compose run --rm backend yarn db:seed + docker-compose "${conf[@]}" run --rm backend yarn db:seed } db_setup() { @@ -89,7 +92,7 @@ db_setup() { } db_stop() { - docker-compose stop db + docker-compose "${conf[@]}" stop db } # Need reliable way to remove db volume and only db volume @@ -117,12 +120,12 @@ db() { test_frontend() { echo "${hr} Running frontend tests ${hr}" - docker-compose run --rm frontend yarn test:ci + docker-compose "${conf[@]}" run --rm frontend yarn test:ci } test_backend() { echo "${hr} Running backend tests ${hr}" - docker-compose run --rm backend yarn test:ci + docker-compose "${conf[@]}" run --rm backend yarn test:ci } # tests plural, because `test` is a builtin shell command @@ -148,12 +151,12 @@ tests() { lint_backend() { echo "${hr} Linting backend ${hr}" - docker-compose run --rm backend yarn lint:ci + docker-compose "${conf[@]}" run --rm backend yarn lint:ci } lint_frontend() { echo "${hr} Linting frontend ${hr}" - docker-compose run --rm frontend yarn lint:ci + docker-compose "${conf[@]}" run --rm frontend yarn lint:ci } lint() { @@ -172,13 +175,36 @@ lint() { } start() { - docker-compose up + docker-compose "${conf[@]}" up } stop() { - docker-compose down + docker-compose "${conf[@]}" down } +# ===== Process env configs + +# We want to specify multiple config files, per +# https://docs.docker.com/compose/extends/#multiple-compose-files +compose_conf() { + local env="${1}" + local envfile="docker-compose.${env}.yml" + + if [[ ! -e "$envfile" ]]; then + echo "$envfile does not appear to exist!" + exit 1 + fi + + configs=( + '-f docker-compose.yml' + "-f docker-compose.${env}.yml" + ) + + echo "${configs[@]}" +} + +# ===== Usage ===== + usage() { # shellcheck disable=SC2086 cat <&2 @@ -192,6 +218,7 @@ EOF # ====== Main ===== main() { + local env="local" # Parse -options while getopts ":e:h" opt; do case $opt in @@ -235,6 +262,8 @@ main() { exit 1 fi + conf=($(compose_conf "$env")) + # Loop over subcommands, look for match with $subcommand. Execute fn if match for i in "${subcommands[@]}"; do if [[ "${i}" == "${subcmd}" ]]; then diff --git a/docker-compose.local.yml b/docker-compose.local.yml new file mode 100644 index 0000000000..0ce8c997b4 --- /dev/null +++ b/docker-compose.local.yml @@ -0,0 +1,12 @@ +version: "3.5" +services: + backend: + environment: + - POSTGRES_HOST=postgres_docker_dev + db: + container_name: postgres_docker_dev + volumes: + - dbdata:/var/lib/postgresql/data + +volumes: + dbdata: diff --git a/docker-compose.test.yml b/docker-compose.test.yml new file mode 100644 index 0000000000..3953f45f96 --- /dev/null +++ b/docker-compose.test.yml @@ -0,0 +1,7 @@ +version: "3.5" +services: + backend: + environment: + - POSTGRES_HOST=postgres_docker_test + db: + container_name: postgres_docker_test diff --git a/docker-compose.yml b/docker-compose.yml index e3fda8a1b9..30e74cd554 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,8 +17,6 @@ services: - "8080:8080" depends_on: - db - environment: - - POSTGRES_HOST=postgres_docker volumes: - ".:/app:rw" frontend: @@ -36,12 +34,9 @@ services: - BACKEND_PROXY=http://backend:8080 db: image: postgres:12.4 - container_name: postgres_docker env_file: .env ports: - "6543:5432" - volumes: - - dbdata:/var/lib/postgresql/data minio: image: minio/minio env_file: .env @@ -57,5 +52,4 @@ services: volumes: - dbdata: {} miniodata: {} From 92a41c4f75a48c57a40498f122a7cf8ccc1e618d Mon Sep 17 00:00:00 2001 From: Daniel Cloud Date: Thu, 28 Jan 2021 12:26:52 -0500 Subject: [PATCH 30/64] bin/docker-run fixes, improvements --- bin/docker-run | 61 +++++++++++++++++++++++++++++--------------------- 1 file changed, 35 insertions(+), 26 deletions(-) diff --git a/bin/docker-run b/bin/docker-run index 533eeaa2a4..0bea731f25 100755 --- a/bin/docker-run +++ b/bin/docker-run @@ -2,14 +2,12 @@ hr="==========" -declare sequelizecli="node_modules/.bin/sequelize" - -declare -a environments=("local" "test") - -declare -a conf +readonly SEQUELIZE_CLI="node_modules/.bin/sequelize" +readonly DEFAULT_ENV='local' +readonly -a ENVIRONMENTS=("$DEFAULT_ENV" "test") # Array of subcommands -declare -a subcommands=( +readonly -a subcommands=( usage setup teardown @@ -20,6 +18,8 @@ declare -a subcommands=( tests ) +declare -a conf + setup() { # First, setup up frontend and backend, install dependencies @@ -55,20 +55,20 @@ db_migrate() { # If 'undo' param provided undo if [[ "undo" == "${option}" ]]; then echo "${hr} Undoing db migrations ${hr}" - docker-compose "${conf[@]}" run --rm backend "$sequelizecli" db:migrate:undo + docker-compose "${conf[@]}" run --rm backend "$SEQUELIZE_CLI" db:migrate:undo exit 0 fi if [[ "status" == "${option}" ]]; then echo "${hr} Checking db migration status ${hr}" - docker-compose "${conf[@]}" run --rm backend "$sequelizecli" db:migrate:status + docker-compose "${conf[@]}" run --rm backend "$SEQUELIZE_CLI" db:migrate:status exit 0 fi # Create/update database tables # yarn docker:db:migrate echo "${hr} Migrating db ${hr}" - docker-compose "${conf[@]}" run --rm backend "$sequelizecli" db:migrate + docker-compose "${conf[@]}" run --rm backend "$SEQUELIZE_CLI" db:migrate } db_seed() { @@ -76,7 +76,7 @@ db_seed() { # If 'undo' param provided undo if [[ "undo" == "${option}" ]]; then echo "${hr} Unseeding db ${hr}" - docker-compose "${conf[@]}" run --rm backend "$sequelizecli" db:seed:undo:all + docker-compose "${conf[@]}" run --rm backend "$SEQUELIZE_CLI" db:seed:undo:all exit 0 fi @@ -217,24 +217,32 @@ EOF # ====== Main ===== +check_env() { + local env_match=0 + for e in "${ENVIRONMENTS[@]}"; do + if [[ "$e" == "${1}" ]]; then + env_match=1; + break; + fi + done + if [[ $env_match -eq 0 ]]; then + echo "Invalid environment '${1}'"; + echo "Valid environments are: ${ENVIRONMENTS[*]}" + exit 1; + fi +} + main() { - local env="local" + docker_env="${DOCKER_ENV:-$DEFAULT_ENV}" + local env_conf="$docker_env" + check_env "$env_conf" + # Parse -options while getopts ":e:h" opt; do case $opt in e) - env="${OPTARG-local}" - local env_match=0 - for e in "${environments[@]}"; do - if [[ "$e" == "$env" ]]; then - env_match=1; - break; - fi - done - if [[ $env_match -eq 0 ]]; then - echo "Invalid environment '${env}'"; - exit 1; - fi + env_conf="${OPTARG:-$env_conf}" + check_env "$env_conf" ;; h) usage @@ -251,7 +259,7 @@ main() { shift $((OPTIND -1)) # Pull first param as subcommand, default to 'usage' if no args provided - local subcmd="${1-usage}"; + local subcmd="${1:-usage}"; # Shift parameters, so $@a has remaining params shift 2>/dev/null @@ -262,7 +270,8 @@ main() { exit 1 fi - conf=($(compose_conf "$env")) + # 'conf' is an array, so we follow https://github.com/koalaman/shellcheck/wiki/SC2207 + IFS=" " read -r -a conf <<< "$(compose_conf "$env_conf")" # Loop over subcommands, look for match with $subcommand. Execute fn if match for i in "${subcommands[@]}"; do @@ -276,4 +285,4 @@ main() { usage } -main "${@-}" +main "$@" From 32f38c88813113731c536cd6430bd273aec799c9 Mon Sep 17 00:00:00 2001 From: Daniel Cloud Date: Thu, 28 Jan 2021 13:27:07 -0500 Subject: [PATCH 31/64] Minor docker-run improvements --- bin/docker-run | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/bin/docker-run b/bin/docker-run index 0bea731f25..e65860aa7a 100755 --- a/bin/docker-run +++ b/bin/docker-run @@ -5,6 +5,7 @@ hr="==========" readonly SEQUELIZE_CLI="node_modules/.bin/sequelize" readonly DEFAULT_ENV='local' readonly -a ENVIRONMENTS=("$DEFAULT_ENV" "test") +readonly DB_SERVICE='db' # Array of subcommands readonly -a subcommands=( @@ -15,7 +16,7 @@ readonly -a subcommands=( stop db lint - tests + 'test' ) declare -a conf @@ -102,6 +103,15 @@ db_stop() { # docker volume prune # } +db_exists() { + # TODO: Less naive detection of database service/image/volume + db_lookup="$(docker-compose images --quiet $DB_SERVICE)" + if [[ -n "$db_lookup" ]]; then + return 1 + fi + return 0 +} + db() { declare -a options=(setup migrate seed stop) option="${1}" @@ -124,12 +134,20 @@ test_frontend() { } test_backend() { + # Check if db_exists, if not setup db + local -i has_db + has_db=$(db_exists) + if [[ has_db -eq 0 ]]; then + echo "${hr} Setting up database prior to testing ${hr}" + db_setup + fi echo "${hr} Running backend tests ${hr}" docker-compose "${conf[@]}" run --rm backend yarn test:ci + db_stop } -# tests plural, because `test` is a builtin shell command -tests() { +# 'run_tests', because `test` is a builtin shell command +run_tests() { declare -a options=(frontend backend) option="${1}" @@ -276,6 +294,9 @@ main() { # Loop over subcommands, look for match with $subcommand. Execute fn if match for i in "${subcommands[@]}"; do if [[ "${i}" == "${subcmd}" ]]; then + if [[ "${subcmd}" == "test" ]]; then + subcmd="run_tests"; + fi "${subcmd}" "$@" exit 0 fi From de13b1febdd9abaf1a5ed9ab87e24aee43b76765 Mon Sep 17 00:00:00 2001 From: Daniel Cloud Date: Fri, 29 Jan 2021 10:40:09 -0500 Subject: [PATCH 32/64] Change 'conf' to be a string, since that's how it gets used. Echo cmds before running them --- bin/docker-run | 70 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 51 insertions(+), 19 deletions(-) diff --git a/bin/docker-run b/bin/docker-run index e65860aa7a..d84e35ebe8 100755 --- a/bin/docker-run +++ b/bin/docker-run @@ -19,8 +19,15 @@ readonly -a subcommands=( 'test' ) -declare -a conf - +# conf should be assigned a string like `-f docker-compose.yml -f docker-compose.locale.yml`/conf +declare conf + +# Expects a command as a string, which it will echo and then execute +echo_exec() { + local cmd="${1}" + echo "$cmd" + $cmd +} setup() { # First, setup up frontend and backend, install dependencies @@ -36,17 +43,19 @@ setup() { setup_backend() { echo "${hr} Setting up backend ${hr}" - docker-compose "${conf[@]}" run --rm backend yarn install + echo_exec "docker-compose ${conf} run --rm backend yarn install" } setup_frontend() { echo "${hr} Setting up frontend ${hr}" - docker-compose "${conf[@]}" run --rm frontend yarn install + echo_exec "docker-compose ${conf} run --rm frontend yarn install" + echo "$cmd" + $cmd } teardown() { docker-compose \ - "${conf[@]}" \ + "${conf}" \ down --volumes } @@ -56,20 +65,26 @@ db_migrate() { # If 'undo' param provided undo if [[ "undo" == "${option}" ]]; then echo "${hr} Undoing db migrations ${hr}" - docker-compose "${conf[@]}" run --rm backend "$SEQUELIZE_CLI" db:migrate:undo + echo_exec "docker-compose ${conf} run --rm backend $SEQUELIZE_CLI db:migrate:undo" + echo "$cmd" + $cmd exit 0 fi if [[ "status" == "${option}" ]]; then echo "${hr} Checking db migration status ${hr}" - docker-compose "${conf[@]}" run --rm backend "$SEQUELIZE_CLI" db:migrate:status + echo_exec "docker-compose ${conf} run --rm backend $SEQUELIZE_CLI db:migrate:status" + echo "$cmd" + $cmd exit 0 fi # Create/update database tables # yarn docker:db:migrate echo "${hr} Migrating db ${hr}" - docker-compose "${conf[@]}" run --rm backend "$SEQUELIZE_CLI" db:migrate + echo_exec "docker-compose ${conf} run --rm backend $SEQUELIZE_CLI db:migrate" + echo "$cmd" + $cmd } db_seed() { @@ -77,14 +92,18 @@ db_seed() { # If 'undo' param provided undo if [[ "undo" == "${option}" ]]; then echo "${hr} Unseeding db ${hr}" - docker-compose "${conf[@]}" run --rm backend "$SEQUELIZE_CLI" db:seed:undo:all + echo_exec "docker-compose ${conf} run --rm backend $SEQUELIZE_CLI db:seed:undo:all" + echo "$cmd" + $cmd exit 0 fi # Populate db with data # yarn docker:db:seed echo "${hr} Seeding database ${hr}" - docker-compose "${conf[@]}" run --rm backend yarn db:seed + echo_exec "docker-compose ${conf} run --rm backend yarn db:seed" + echo "$cmd" + $cmd } db_setup() { @@ -93,7 +112,9 @@ db_setup() { } db_stop() { - docker-compose "${conf[@]}" stop db + echo_exec "docker-compose ${conf} stop db" + echo "$cmd" + $cmd } # Need reliable way to remove db volume and only db volume @@ -130,7 +151,9 @@ db() { test_frontend() { echo "${hr} Running frontend tests ${hr}" - docker-compose "${conf[@]}" run --rm frontend yarn test:ci + echo_exec "docker-compose ${conf} run --rm frontend yarn test:ci" + echo "$cmd" + $cmd } test_backend() { @@ -142,7 +165,9 @@ test_backend() { db_setup fi echo "${hr} Running backend tests ${hr}" - docker-compose "${conf[@]}" run --rm backend yarn test:ci + echo_exec "docker-compose ${conf} run --rm backend yarn test:ci" + echo "$cmd" + $cmd db_stop } @@ -169,12 +194,16 @@ run_tests() { lint_backend() { echo "${hr} Linting backend ${hr}" - docker-compose "${conf[@]}" run --rm backend yarn lint:ci + echo_exec "docker-compose ${conf} run --rm backend yarn lint:ci" + echo "$cmd" + $cmd } lint_frontend() { echo "${hr} Linting frontend ${hr}" - docker-compose "${conf[@]}" run --rm frontend yarn lint:ci + echo_exec "docker-compose ${conf} run --rm frontend yarn lint:ci" + echo "$cmd" + $cmd } lint() { @@ -193,11 +222,15 @@ lint() { } start() { - docker-compose "${conf[@]}" up + echo_exec "docker-compose ${conf} up" + echo "$cmd" + $cmd } stop() { - docker-compose "${conf[@]}" down + echo_exec "docker-compose ${conf} down" + echo "$cmd" + $cmd } # ===== Process env configs @@ -288,8 +321,7 @@ main() { exit 1 fi - # 'conf' is an array, so we follow https://github.com/koalaman/shellcheck/wiki/SC2207 - IFS=" " read -r -a conf <<< "$(compose_conf "$env_conf")" + conf=$(compose_conf "$env_conf") # Loop over subcommands, look for match with $subcommand. Execute fn if match for i in "${subcommands[@]}"; do From a232a9155c190fbdc5acf1cbe6484803f1e40072 Mon Sep 17 00:00:00 2001 From: Daniel Cloud Date: Fri, 29 Jan 2021 12:48:13 -0500 Subject: [PATCH 33/64] docker-run: 'test' uses test env by default --- bin/docker-run | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/bin/docker-run b/bin/docker-run index d84e35ebe8..46f51db433 100755 --- a/bin/docker-run +++ b/bin/docker-run @@ -4,7 +4,8 @@ hr="==========" readonly SEQUELIZE_CLI="node_modules/.bin/sequelize" readonly DEFAULT_ENV='local' -readonly -a ENVIRONMENTS=("$DEFAULT_ENV" "test") +readonly TEST_ENV='test' +readonly -a ENVIRONMENTS=("$DEFAULT_ENV" "$TEST_ENV") readonly DB_SERVICE='db' # Array of subcommands @@ -21,6 +22,7 @@ readonly -a subcommands=( # conf should be assigned a string like `-f docker-compose.yml -f docker-compose.locale.yml`/conf declare conf +declare -i env_override=0 # Expects a command as a string, which it will echo and then execute echo_exec() { @@ -112,7 +114,7 @@ db_setup() { } db_stop() { - echo_exec "docker-compose ${conf} stop db" + echo_exec "docker-compose ${conf} stop db" echo "$cmd" $cmd } @@ -252,6 +254,7 @@ compose_conf() { ) echo "${configs[@]}" + return 0 } # ===== Usage ===== @@ -285,6 +288,9 @@ check_env() { main() { docker_env="${DOCKER_ENV:-$DEFAULT_ENV}" + if [[ -n "$DOCKER_ENV" ]]; then + env_override=1 + fi local env_conf="$docker_env" check_env "$env_conf" @@ -294,6 +300,7 @@ main() { e) env_conf="${OPTARG:-$env_conf}" check_env "$env_conf" + env_override=1 ;; h) usage @@ -327,6 +334,12 @@ main() { for i in "${subcommands[@]}"; do if [[ "${i}" == "${subcmd}" ]]; then if [[ "${subcmd}" == "test" ]]; then + if [[ $env_override -eq 0 ]]; then + echo "${hr}> Test commands use 'test' config, unless overridden <${hr}" + conf=$(compose_conf "$TEST_ENV") + else + echo "Running tests in '$env_conf'" + fi subcmd="run_tests"; fi "${subcmd}" "$@" From ac43405c04826797c2fe7bce2b4f55b23deafda6 Mon Sep 17 00:00:00 2001 From: Daniel Cloud Date: Fri, 29 Jan 2021 16:19:07 -0500 Subject: [PATCH 34/64] Undo the composed compose files; docker-compose wasn't handling differend db instances differently --- bin/docker-run | 26 +++++++++-------------- docker-compose.local.yml | 12 ----------- docker-compose.test.yml | 45 +++++++++++++++++++++++++++++++++++++++- docker-compose.yml | 6 ++++++ 4 files changed, 60 insertions(+), 29 deletions(-) delete mode 100644 docker-compose.local.yml diff --git a/bin/docker-run b/bin/docker-run index 46f51db433..ac40ce54d0 100755 --- a/bin/docker-run +++ b/bin/docker-run @@ -120,11 +120,10 @@ db_stop() { } # Need reliable way to remove db volume and only db volume -# db_destroy() { -# db_stop -# docker-compose rm -v -- db -# docker volume prune -# } +db_destroy() { + echo "${hr} Destroying db ${hr}" + echo_exec "docker-compose ${conf} down --volumes" +} db_exists() { # TODO: Less naive detection of database service/image/volume @@ -154,8 +153,6 @@ db() { test_frontend() { echo "${hr} Running frontend tests ${hr}" echo_exec "docker-compose ${conf} run --rm frontend yarn test:ci" - echo "$cmd" - $cmd } test_backend() { @@ -168,9 +165,7 @@ test_backend() { fi echo "${hr} Running backend tests ${hr}" echo_exec "docker-compose ${conf} run --rm backend yarn test:ci" - echo "$cmd" - $cmd - db_stop + db_destroy } # 'run_tests', because `test` is a builtin shell command @@ -243,17 +238,16 @@ compose_conf() { local env="${1}" local envfile="docker-compose.${env}.yml" + if [[ "$env" == "local" ]]; then + envfile="docker-compose.yml" + fi + if [[ ! -e "$envfile" ]]; then echo "$envfile does not appear to exist!" exit 1 fi - configs=( - '-f docker-compose.yml' - "-f docker-compose.${env}.yml" - ) - - echo "${configs[@]}" + echo "-f ${envfile}" return 0 } diff --git a/docker-compose.local.yml b/docker-compose.local.yml deleted file mode 100644 index 0ce8c997b4..0000000000 --- a/docker-compose.local.yml +++ /dev/null @@ -1,12 +0,0 @@ -version: "3.5" -services: - backend: - environment: - - POSTGRES_HOST=postgres_docker_dev - db: - container_name: postgres_docker_dev - volumes: - - dbdata:/var/lib/postgresql/data - -volumes: - dbdata: diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 3953f45f96..1776c0854b 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -1,7 +1,50 @@ version: "3.5" services: backend: + build: + context: . + command: yarn server + user: ${CURRENT_USER:-root} + ports: + - "8080:8080" + depends_on: + - testdb environment: - POSTGRES_HOST=postgres_docker_test - db: + volumes: + - ".:/app:rw" + frontend: + build: + context: . + command: yarn start + user: ${CURRENT_USER:-root} + stdin_open: true + ports: + - "3000:3000" + volumes: + - "./frontend:/app:rw" + - "./scripts:/app/scripts" + environment: + - BACKEND_PROXY=http://backend:8080 + testdb: + image: postgres:12.4 container_name: postgres_docker_test + env_file: .env + ports: + - "6543:5432" + minio: + image: minio/minio + env_file: .env + ports: + - "9000:9000" + volumes: + - miniodata:/data + command: server /data + aws-cli: + image: amazon/aws-cli + env_file: .env + command: ["--endpoint-url", "$S3_ENDPOINT", "s3api", "create-bucket", "--bucket", "$S3_BUCKET"] + + +volumes: + miniodata: {} diff --git a/docker-compose.yml b/docker-compose.yml index 30e74cd554..e3fda8a1b9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,6 +17,8 @@ services: - "8080:8080" depends_on: - db + environment: + - POSTGRES_HOST=postgres_docker volumes: - ".:/app:rw" frontend: @@ -34,9 +36,12 @@ services: - BACKEND_PROXY=http://backend:8080 db: image: postgres:12.4 + container_name: postgres_docker env_file: .env ports: - "6543:5432" + volumes: + - dbdata:/var/lib/postgresql/data minio: image: minio/minio env_file: .env @@ -52,4 +57,5 @@ services: volumes: + dbdata: {} miniodata: {} From 188b05f75d63755454204f48878f95e8bd2ef8f6 Mon Sep 17 00:00:00 2001 From: Daniel Cloud Date: Fri, 29 Jan 2021 16:20:16 -0500 Subject: [PATCH 35/64] Remove 2-liners in favor of echo_exec function --- bin/docker-run | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/bin/docker-run b/bin/docker-run index ac40ce54d0..8061cbfdae 100755 --- a/bin/docker-run +++ b/bin/docker-run @@ -51,8 +51,6 @@ setup_backend() { setup_frontend() { echo "${hr} Setting up frontend ${hr}" echo_exec "docker-compose ${conf} run --rm frontend yarn install" - echo "$cmd" - $cmd } teardown() { @@ -68,16 +66,12 @@ db_migrate() { if [[ "undo" == "${option}" ]]; then echo "${hr} Undoing db migrations ${hr}" echo_exec "docker-compose ${conf} run --rm backend $SEQUELIZE_CLI db:migrate:undo" - echo "$cmd" - $cmd exit 0 fi if [[ "status" == "${option}" ]]; then echo "${hr} Checking db migration status ${hr}" echo_exec "docker-compose ${conf} run --rm backend $SEQUELIZE_CLI db:migrate:status" - echo "$cmd" - $cmd exit 0 fi @@ -85,8 +79,6 @@ db_migrate() { # yarn docker:db:migrate echo "${hr} Migrating db ${hr}" echo_exec "docker-compose ${conf} run --rm backend $SEQUELIZE_CLI db:migrate" - echo "$cmd" - $cmd } db_seed() { @@ -95,8 +87,6 @@ db_seed() { if [[ "undo" == "${option}" ]]; then echo "${hr} Unseeding db ${hr}" echo_exec "docker-compose ${conf} run --rm backend $SEQUELIZE_CLI db:seed:undo:all" - echo "$cmd" - $cmd exit 0 fi @@ -104,8 +94,6 @@ db_seed() { # yarn docker:db:seed echo "${hr} Seeding database ${hr}" echo_exec "docker-compose ${conf} run --rm backend yarn db:seed" - echo "$cmd" - $cmd } db_setup() { @@ -115,8 +103,6 @@ db_setup() { db_stop() { echo_exec "docker-compose ${conf} stop db" - echo "$cmd" - $cmd } # Need reliable way to remove db volume and only db volume @@ -192,15 +178,11 @@ run_tests() { lint_backend() { echo "${hr} Linting backend ${hr}" echo_exec "docker-compose ${conf} run --rm backend yarn lint:ci" - echo "$cmd" - $cmd } lint_frontend() { echo "${hr} Linting frontend ${hr}" echo_exec "docker-compose ${conf} run --rm frontend yarn lint:ci" - echo "$cmd" - $cmd } lint() { @@ -220,14 +202,10 @@ lint() { start() { echo_exec "docker-compose ${conf} up" - echo "$cmd" - $cmd } stop() { echo_exec "docker-compose ${conf} down" - echo "$cmd" - $cmd } # ===== Process env configs From ece7bb4b25a2f4fb920435b48e569198bf190405 Mon Sep 17 00:00:00 2001 From: Daniel Cloud Date: Fri, 29 Jan 2021 16:40:59 -0500 Subject: [PATCH 36/64] docker-run: Make default config a readonly var --- bin/docker-run | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/docker-run b/bin/docker-run index 8061cbfdae..f5aed4bb72 100755 --- a/bin/docker-run +++ b/bin/docker-run @@ -3,6 +3,7 @@ hr="==========" readonly SEQUELIZE_CLI="node_modules/.bin/sequelize" +readonly DEFAULT_CONFIG='docker-compose.yml' readonly DEFAULT_ENV='local' readonly TEST_ENV='test' readonly -a ENVIRONMENTS=("$DEFAULT_ENV" "$TEST_ENV") @@ -217,7 +218,7 @@ compose_conf() { local envfile="docker-compose.${env}.yml" if [[ "$env" == "local" ]]; then - envfile="docker-compose.yml" + envfile="$DEFAULT_CONFIG" fi if [[ ! -e "$envfile" ]]; then From c8886f0d4364cb21e2af2d683d815bf90e8365ca Mon Sep 17 00:00:00 2001 From: Daniel Cloud Date: Mon, 1 Feb 2021 17:18:35 -0500 Subject: [PATCH 37/64] Add simpler run-tests script. Improve docker-compose.test.yml with namespaced services, remove minodata volume. --- bin/run-tests | 11 +++++++++++ docker-compose.test.yml | 29 ++++++++++++++++++----------- 2 files changed, 29 insertions(+), 11 deletions(-) create mode 100755 bin/run-tests diff --git a/bin/run-tests b/bin/run-tests new file mode 100755 index 0000000000..a1b7f9a63e --- /dev/null +++ b/bin/run-tests @@ -0,0 +1,11 @@ +#!/bin/bash + +echo "Running tests in using test config 'docker-compose.test.yml'" + +docker-compose \ + -f 'docker-compose.test.yml' \ + run test-backend bash -c "yarn db:migrate; yarn db:seed; yarn test:ci" +wait +docker-compose \ + -f 'docker-compose.test.yml' \ + down --volumes diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 1776c0854b..8ffc9491db 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -1,6 +1,6 @@ version: "3.5" services: - backend: + test-backend: build: context: . command: yarn server @@ -8,12 +8,14 @@ services: ports: - "8080:8080" depends_on: - - testdb + - test-db environment: - POSTGRES_HOST=postgres_docker_test volumes: - ".:/app:rw" - frontend: + networks: + - test + test-frontend: build: context: . command: yarn start @@ -25,26 +27,31 @@ services: - "./frontend:/app:rw" - "./scripts:/app/scripts" environment: - - BACKEND_PROXY=http://backend:8080 - testdb: + - BACKEND_PROXY=http://test-backend:8080 + networks: + - test + test-db: image: postgres:12.4 container_name: postgres_docker_test env_file: .env ports: - "6543:5432" - minio: + networks: + - test + test-minio: image: minio/minio env_file: .env ports: - "9000:9000" - volumes: - - miniodata:/data command: server /data + networks: + - test aws-cli: image: amazon/aws-cli env_file: .env command: ["--endpoint-url", "$S3_ENDPOINT", "s3api", "create-bucket", "--bucket", "$S3_BUCKET"] + networks: + - test - -volumes: - miniodata: {} +networks: + test: From 8b0beced865b3a9733bb5c61123dcc9d33451981 Mon Sep 17 00:00:00 2001 From: Daniel Cloud Date: Mon, 1 Feb 2021 17:20:07 -0500 Subject: [PATCH 38/64] Remove bin/docker-run since we are no longer composing multiple docker-compose files. Small scripts from here on out --- bin/docker-run | 327 ------------------------------------------------- 1 file changed, 327 deletions(-) delete mode 100755 bin/docker-run diff --git a/bin/docker-run b/bin/docker-run deleted file mode 100755 index f5aed4bb72..0000000000 --- a/bin/docker-run +++ /dev/null @@ -1,327 +0,0 @@ -#!/bin/bash - -hr="==========" - -readonly SEQUELIZE_CLI="node_modules/.bin/sequelize" -readonly DEFAULT_CONFIG='docker-compose.yml' -readonly DEFAULT_ENV='local' -readonly TEST_ENV='test' -readonly -a ENVIRONMENTS=("$DEFAULT_ENV" "$TEST_ENV") -readonly DB_SERVICE='db' - -# Array of subcommands -readonly -a subcommands=( - usage - setup - teardown - start - stop - db - lint - 'test' -) - -# conf should be assigned a string like `-f docker-compose.yml -f docker-compose.locale.yml`/conf -declare conf -declare -i env_override=0 - -# Expects a command as a string, which it will echo and then execute -echo_exec() { - local cmd="${1}" - echo "$cmd" - $cmd -} - -setup() { - # First, setup up frontend and backend, install dependencies - # yarn docker:deps - setup_backend - setup_frontend - - db_setup "" - - # Stop the db manually? - # db_stop -} - -setup_backend() { - echo "${hr} Setting up backend ${hr}" - echo_exec "docker-compose ${conf} run --rm backend yarn install" -} - -setup_frontend() { - echo "${hr} Setting up frontend ${hr}" - echo_exec "docker-compose ${conf} run --rm frontend yarn install" -} - -teardown() { - docker-compose \ - "${conf}" \ - down --volumes -} - -db_migrate() { - local -a options=(status undo) - local option="${1}" - # If 'undo' param provided undo - if [[ "undo" == "${option}" ]]; then - echo "${hr} Undoing db migrations ${hr}" - echo_exec "docker-compose ${conf} run --rm backend $SEQUELIZE_CLI db:migrate:undo" - exit 0 - fi - - if [[ "status" == "${option}" ]]; then - echo "${hr} Checking db migration status ${hr}" - echo_exec "docker-compose ${conf} run --rm backend $SEQUELIZE_CLI db:migrate:status" - exit 0 - fi - - # Create/update database tables - # yarn docker:db:migrate - echo "${hr} Migrating db ${hr}" - echo_exec "docker-compose ${conf} run --rm backend $SEQUELIZE_CLI db:migrate" -} - -db_seed() { - local option="${1}" - # If 'undo' param provided undo - if [[ "undo" == "${option}" ]]; then - echo "${hr} Unseeding db ${hr}" - echo_exec "docker-compose ${conf} run --rm backend $SEQUELIZE_CLI db:seed:undo:all" - exit 0 - fi - - # Populate db with data - # yarn docker:db:seed - echo "${hr} Seeding database ${hr}" - echo_exec "docker-compose ${conf} run --rm backend yarn db:seed" -} - -db_setup() { - db_migrate "" - db_seed "" -} - -db_stop() { - echo_exec "docker-compose ${conf} stop db" -} - -# Need reliable way to remove db volume and only db volume -db_destroy() { - echo "${hr} Destroying db ${hr}" - echo_exec "docker-compose ${conf} down --volumes" -} - -db_exists() { - # TODO: Less naive detection of database service/image/volume - db_lookup="$(docker-compose images --quiet $DB_SERVICE)" - if [[ -n "$db_lookup" ]]; then - return 1 - fi - return 0 -} - -db() { - declare -a options=(setup migrate seed stop) - option="${1}" - for i in "${options[@]}"; do - if [ "${i}" == "${option}" ]; then - shift 2>/dev/null - "db_${option}" "$@" - exit 0 - fi - done - - # If no matching option, print options - echo "db expects one of the following: ${options[*]}" - exit 1 -} - -test_frontend() { - echo "${hr} Running frontend tests ${hr}" - echo_exec "docker-compose ${conf} run --rm frontend yarn test:ci" -} - -test_backend() { - # Check if db_exists, if not setup db - local -i has_db - has_db=$(db_exists) - if [[ has_db -eq 0 ]]; then - echo "${hr} Setting up database prior to testing ${hr}" - db_setup - fi - echo "${hr} Running backend tests ${hr}" - echo_exec "docker-compose ${conf} run --rm backend yarn test:ci" - db_destroy -} - -# 'run_tests', because `test` is a builtin shell command -run_tests() { - declare -a options=(frontend backend) - option="${1}" - - # Check if user specified 'frontend' or 'backend' option - if [[ -n $option ]]; then - for i in "${options[@]}"; do - if [ "${i}" == "${option}" ]; then - "test_${option}" - exit 0 - fi - done - fi - - # Otherwise, run both - test_frontend - test_backend - -} - -lint_backend() { - echo "${hr} Linting backend ${hr}" - echo_exec "docker-compose ${conf} run --rm backend yarn lint:ci" -} - -lint_frontend() { - echo "${hr} Linting frontend ${hr}" - echo_exec "docker-compose ${conf} run --rm frontend yarn lint:ci" -} - -lint() { - local -a options=(frontend backend) - option="${1}" - - for i in "${options[@]}"; do - if [[ "${i}" == "$option" ]]; then - "lint_${option}" - exit 0 - fi - done - - lint_frontend - lint_backend -} - -start() { - echo_exec "docker-compose ${conf} up" -} - -stop() { - echo_exec "docker-compose ${conf} down" -} - -# ===== Process env configs - -# We want to specify multiple config files, per -# https://docs.docker.com/compose/extends/#multiple-compose-files -compose_conf() { - local env="${1}" - local envfile="docker-compose.${env}.yml" - - if [[ "$env" == "local" ]]; then - envfile="$DEFAULT_CONFIG" - fi - - if [[ ! -e "$envfile" ]]; then - echo "$envfile does not appear to exist!" - exit 1 - fi - - echo "-f ${envfile}" - return 0 -} - -# ===== Usage ===== - -usage() { - # shellcheck disable=SC2086 - cat <&2 -Run commands in Docker containers -Usage: $0 [-h] [-e env] -Subcommands: - ${subcommands[*]} -EOF -} - -# ====== Main ===== - -check_env() { - local env_match=0 - for e in "${ENVIRONMENTS[@]}"; do - if [[ "$e" == "${1}" ]]; then - env_match=1; - break; - fi - done - if [[ $env_match -eq 0 ]]; then - echo "Invalid environment '${1}'"; - echo "Valid environments are: ${ENVIRONMENTS[*]}" - exit 1; - fi -} - -main() { - docker_env="${DOCKER_ENV:-$DEFAULT_ENV}" - if [[ -n "$DOCKER_ENV" ]]; then - env_override=1 - fi - local env_conf="$docker_env" - check_env "$env_conf" - - # Parse -options - while getopts ":e:h" opt; do - case $opt in - e) - env_conf="${OPTARG:-$env_conf}" - check_env "$env_conf" - env_override=1 - ;; - h) - usage - exit - ;; - \?) - echo "Invalid option" >&2 - echo - ;; - esac - done - - # After parsing options, shift so $@ has only params - shift $((OPTIND -1)) - - # Pull first param as subcommand, default to 'usage' if no args provided - local subcmd="${1:-usage}"; - - # Shift parameters, so $@a has remaining params - shift 2>/dev/null - - # If subcommand is empty string, print usage - if [ -z "$subcmd" ]; then - usage - exit 1 - fi - - conf=$(compose_conf "$env_conf") - - # Loop over subcommands, look for match with $subcommand. Execute fn if match - for i in "${subcommands[@]}"; do - if [[ "${i}" == "${subcmd}" ]]; then - if [[ "${subcmd}" == "test" ]]; then - if [[ $env_override -eq 0 ]]; then - echo "${hr}> Test commands use 'test' config, unless overridden <${hr}" - conf=$(compose_conf "$TEST_ENV") - else - echo "Running tests in '$env_conf'" - fi - subcmd="run_tests"; - fi - "${subcmd}" "$@" - exit 0 - fi - done - - # If we didn't run a subcommand and exit already, print usage - usage -} - -main "$@" From 387a5e70601ba69f1d87a2d842e4a97be5893df9 Mon Sep 17 00:00:00 2001 From: Chuck McAndrew Date: Thu, 28 Jan 2021 14:43:03 -0500 Subject: [PATCH 39/64] rebase on main --- frontend/src/fetchers/File.js | 1 - 1 file changed, 1 deletion(-) diff --git a/frontend/src/fetchers/File.js b/frontend/src/fetchers/File.js index 275d96f6c0..815a012222 100644 --- a/frontend/src/fetchers/File.js +++ b/frontend/src/fetchers/File.js @@ -1,5 +1,4 @@ import join from 'url-join'; -import { get } from './index'; const activityReportUrl = join('/', 'api', 'files'); From 43e18c98b6b383976cd56c7396acb5b8d3756a24 Mon Sep 17 00:00:00 2001 From: Daniel Cloud Date: Mon, 1 Feb 2021 17:25:35 -0500 Subject: [PATCH 40/64] One last rename --- .circleci/config.yml | 12 ++++++------ docker-compose-dss.yml => docker-compose.dss.yml | 0 2 files changed, 6 insertions(+), 6 deletions(-) rename docker-compose-dss.yml => docker-compose.dss.yml (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7265c80d1f..4141476207 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -240,12 +240,12 @@ jobs: - run: name: Start up local server command: | # production style build (single BE server with static FE) - docker-compose -f docker-compose-dss.yml run --rm server yarn install --production=false - docker-compose -f docker-compose-dss.yml run --rm server yarn --cwd frontend install --production=false - docker-compose -f docker-compose-dss.yml run --rm server yarn build - docker-compose -f docker-compose-dss.yml run --rm server yarn --cwd frontend run build - docker-compose -f docker-compose-dss.yml up -d - docker-compose -f docker-compose-dss.yml exec server yarn db:migrate:ci + docker-compose -f docker-compose.dss.yml run --rm server yarn install --production=false + docker-compose -f docker-compose.dss.yml run --rm server yarn --cwd frontend install --production=false + docker-compose -f docker-compose.dss.yml run --rm server yarn build + docker-compose -f docker-compose.dss.yml run --rm server yarn --cwd frontend run build + docker-compose -f docker-compose.dss.yml up -d + docker-compose -f docker-compose.dss.yml exec server yarn db:migrate:ci - run: name: Pull OWASP ZAP docker image command: docker pull owasp/zap2docker-weekly diff --git a/docker-compose-dss.yml b/docker-compose.dss.yml similarity index 100% rename from docker-compose-dss.yml rename to docker-compose.dss.yml From 291fad8e203248f6e3668c910b65b477c6916b04 Mon Sep 17 00:00:00 2001 From: Chuck McAndrew Date: Thu, 28 Jan 2021 16:52:33 -0500 Subject: [PATCH 41/64] fmt --- frontend/src/components/FileUploader.js | 84 +++++++++---------------- 1 file changed, 29 insertions(+), 55 deletions(-) diff --git a/frontend/src/components/FileUploader.js b/frontend/src/components/FileUploader.js index 3582f60de9..bb27f886bb 100644 --- a/frontend/src/components/FileUploader.js +++ b/frontend/src/components/FileUploader.js @@ -8,11 +8,9 @@ import React from 'react'; import PropTypes from 'prop-types'; import { useDropzone } from 'react-dropzone'; -import * as FS from 'fs'; - import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { faTrash } from '@fortawesome/free-solid-svg-icons'; -import { Tag, Button, Grid } from '@trussworks/react-uswds'; +import { Button } from '@trussworks/react-uswds'; import { uploadFile } from '../fetchers/File'; import './FileUploader.css'; @@ -23,57 +21,31 @@ function Dropzone(props) { let attachmentType; if (id === 'attachments') { attachmentType = 'ATTACHMENT'; - } else if (id === "other-resources") { + } else if (id === 'other-resources') { attachmentType = 'RESOURCE'; } e.forEach( async (file) => { - try { - const data = new FormData() - data.append("reportId", reportId) - data.append("attachmentType", attachmentType) - data.append("file", file) - await uploadFile(data) - } catch (error) { - console.log(error) - } - + try { + const data = new FormData(); + data.append('reportId', reportId); + data.append('attachmentType', attachmentType); + data.append('file', file) + await uploadFile(data) + } catch (error) { + // eslint-disable-next-line no-console + console.log(error); + } }); - - console.log(e) onChange(e); }; const { getRootProps, getInputProps } = useDropzone({ onDrop }); - // I tried moving these styles to a css file and applying a class to the container - // and span. The styles were not being applied, it seems like the Dropzone library - // is messing with the styles somewhere - const containerStyle = { - maxWidth: '21rem', - height: '8rem', - display: 'flex', - justifyContent: 'center', - alignItems: 'center', - borderStyle: 'dashed', - borderWidth: '0.125rem', - borderColor: '#979797', - }; - - const textStyle = { - textAlign: 'left', - fontSize: '1.25rem', - backgroundColor: "#0166AB", - color: "white", - padding: ".5rem", - borderRadius: "5px", - }; - - return (
-
@@ -82,35 +54,37 @@ function Dropzone(props) { Dropzone.propTypes = { onChange: PropTypes.func.isRequired, + reportId: PropTypes.any.isRequired, + id: PropTypes.string.isRequired, }; const FileTable = ({onFileRemoved, files}) => { let msg if (files.length === 0) { msg = ( -

No files uploaded

+

No files uploaded

) } return ( -
- - -
+
+ + + - - - {files.map((file, index) => ( - From ab3b87a9f0f04426cae5a1306cf4da1612cb719c Mon Sep 17 00:00:00 2001 From: Chuck McAndrew Date: Thu, 28 Jan 2021 17:25:17 -0500 Subject: [PATCH 42/64] fmt --- frontend/src/components/FileUploader.js | 118 ++++++++++-------- frontend/src/components/Navigator/index.js | 1 + frontend/src/fetchers/File.js | 6 +- .../ActivityReport/Pages/topicsResources.js | 1 + 4 files changed, 69 insertions(+), 57 deletions(-) diff --git a/frontend/src/components/FileUploader.js b/frontend/src/components/FileUploader.js index bb27f886bb..8c7deab3cf 100644 --- a/frontend/src/components/FileUploader.js +++ b/frontend/src/components/FileUploader.js @@ -10,8 +10,8 @@ import PropTypes from 'prop-types'; import { useDropzone } from 'react-dropzone'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { faTrash } from '@fortawesome/free-solid-svg-icons'; -import { Button } from '@trussworks/react-uswds'; -import { uploadFile } from '../fetchers/File'; +import { Button } from '@trussworks/react-uswds'; +import uploadFile from '../fetchers/File'; import './FileUploader.css'; @@ -24,13 +24,13 @@ function Dropzone(props) { } else if (id === 'other-resources') { attachmentType = 'RESOURCE'; } - e.forEach( async (file) => { + e.forEach(async (file) => { try { const data = new FormData(); data.append('reportId', reportId); data.append('attachmentType', attachmentType); - data.append('file', file) - await uploadFile(data) + data.append('file', file); + await uploadFile(data); } catch (error) { // eslint-disable-next-line no-console console.log(error); @@ -45,7 +45,7 @@ function Dropzone(props) { {...getRootProps()} > - @@ -54,69 +54,77 @@ function Dropzone(props) { Dropzone.propTypes = { onChange: PropTypes.func.isRequired, - reportId: PropTypes.any.isRequired, + reportId: PropTypes.node.isRequired, id: PropTypes.string.isRequired, }; -const FileTable = ({onFileRemoved, files}) => { - let msg +const FileTable = ({ onFileRemoved, files }) => { + let msg; if (files.length === 0) { msg = ( -

No files uploaded

- ) +

No files uploaded

+ ); } return ( -
-
Name + Size + Status +
+ {file.name} @@ -121,14 +95,14 @@ const FileTable = ({onFileRemoved, files}) => {
- - - - - +
+
- Name - - Size - - Status - -
+ + + + + - {files.map((file, index) => ( - - - - - + {files.map((file, index) => ( + + + + + - + - ))} + ))}
+ Name + + Size + + Status +
- {file.name} - - {`${(file.size / 1000).toFixed(1) } KB`} - - Uploaded - - -
+ {file.name} + + {`${(file.size / 1000).toFixed(1)} KB`} + + Uploaded + + +
{ msg }
); -} -const FileUploader = ({ onChange, files, reportId, id }) => { +}; +FileTable.propTypes = { + onFileRemoved: PropTypes.func.isRequired, + files: PropTypes.arrayOf(PropTypes.instanceOf(File)), +}; +FileTable.defaultProps = { + files: [], +}; +const FileUploader = ({ + onChange, files, reportId, id, +}) => { const onFilesAdded = (newFiles) => { onChange([...files, ...newFiles]); }; @@ -129,7 +137,7 @@ const FileUploader = ({ onChange, files, reportId, id }) => { <> - + ); }; @@ -137,6 +145,8 @@ const FileUploader = ({ onChange, files, reportId, id }) => { FileUploader.propTypes = { onChange: PropTypes.func.isRequired, files: PropTypes.arrayOf(PropTypes.instanceOf(File)), + reportId: PropTypes.node.isRequired, + id: PropTypes.string.isRequired, }; FileUploader.defaultProps = { diff --git a/frontend/src/components/Navigator/index.js b/frontend/src/components/Navigator/index.js index 9a5e59a53c..a0cf8a44b0 100644 --- a/frontend/src/components/Navigator/index.js +++ b/frontend/src/components/Navigator/index.js @@ -172,6 +172,7 @@ Navigator.propTypes = { currentPage: PropTypes.string.isRequired, autoSaveInterval: PropTypes.number, additionalData: PropTypes.shape({}), + reportId: PropTypes.node.isRequired, }; Navigator.defaultProps = { diff --git a/frontend/src/fetchers/File.js b/frontend/src/fetchers/File.js index 815a012222..a1a2f176b4 100644 --- a/frontend/src/fetchers/File.js +++ b/frontend/src/fetchers/File.js @@ -2,14 +2,14 @@ import join from 'url-join'; const activityReportUrl = join('/', 'api', 'files'); -export const uploadFile = async (data) => { +export default async function uploadFile(data) { const res = await fetch(activityReportUrl, { method: 'POST', credentials: 'same-origin', - body: data + body: data, }); if (!res.ok) { throw new Error(res.statusText); } return res; -}; \ No newline at end of file +} diff --git a/frontend/src/pages/ActivityReport/Pages/topicsResources.js b/frontend/src/pages/ActivityReport/Pages/topicsResources.js index 2bc4cb9293..994aacccd1 100644 --- a/frontend/src/pages/ActivityReport/Pages/topicsResources.js +++ b/frontend/src/pages/ActivityReport/Pages/topicsResources.js @@ -81,6 +81,7 @@ TopicsResources.propTypes = { register: PropTypes.func.isRequired, // eslint-disable-next-line react/forbid-prop-types control: PropTypes.object.isRequired, + reportId: PropTypes.node.isRequired, }; const sections = [ From b4c0aab31aa5d8397c8a57b77c3000600999b974 Mon Sep 17 00:00:00 2001 From: Chuck McAndrew Date: Fri, 29 Jan 2021 09:27:29 -0500 Subject: [PATCH 43/64] fix tests --- frontend/src/components/FileUploader.js | 7 +++++-- frontend/src/components/__tests__/FileUploader.js | 11 ++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/frontend/src/components/FileUploader.js b/frontend/src/components/FileUploader.js index 8c7deab3cf..a6a027e66a 100644 --- a/frontend/src/components/FileUploader.js +++ b/frontend/src/components/FileUploader.js @@ -67,8 +67,9 @@ const FileTable = ({ onFileRemoved, files }) => { } return (
- +
+ @@ -79,10 +80,12 @@ const FileTable = ({ onFileRemoved, files }) => { Status {files.map((file, index) => ( - + diff --git a/frontend/src/components/__tests__/FileUploader.js b/frontend/src/components/__tests__/FileUploader.js index e9f3ca4fc6..7498fba101 100644 --- a/frontend/src/components/__tests__/FileUploader.js +++ b/frontend/src/components/__tests__/FileUploader.js @@ -34,7 +34,7 @@ describe('FileUploader', () => { it('onDrop adds calls the onChange method', async () => { const mockOnChange = jest.fn(); const data = mockData([file('file')]); - const ui = ; + const ui = ; const { container, rerender } = render(ui); const dropzone = container.querySelector('div'); @@ -45,16 +45,17 @@ describe('FileUploader', () => { }); it('files are properly displayed', () => { - render( {}} files={[file('fileOne'), file('fileTwo')]} />); + render( {}} files={[file('fileOne'), file('fileTwo')]} />); expect(screen.getByText('fileOne')).toBeVisible(); expect(screen.getByText('fileTwo')).toBeVisible(); }); it('files can be removed', () => { const mockOnChange = jest.fn(); - render(); - const fileOne = screen.getByText('fileOne'); - fireEvent.click(fileOne.nextSibling); + render(); + const fileTwo = screen.getByText('fileTwo'); + console.log(fileTwo.parentNode.lastChild.firstChild) + fireEvent.click(fileTwo.parentNode.lastChild.firstChild); expect(mockOnChange).toHaveBeenCalledWith([file('fileTwo')]); }); From 1ecd7947a98f9407287596eec71f414d928df2a5 Mon Sep 17 00:00:00 2001 From: Chuck McAndrew Date: Fri, 29 Jan 2021 17:10:16 -0500 Subject: [PATCH 44/64] update so attached files display --- frontend/src/components/FileUploader.js | 12 +++++------ .../src/components/__tests__/FileUploader.js | 4 ++-- .../ActivityReport/Pages/topicsResources.js | 8 +++---- .../20210129210854-add-fileSize-to-files.js | 14 +++++++++++++ src/models/activityReport.js | 3 ++- src/models/file.js | 4 ++++ src/routes/files/handlers.js | 15 +++++++++++-- src/routes/files/handlers.test.js | 10 +++++++++ src/services/activityReports.js | 21 ++++++++++++++++++- 9 files changed, 75 insertions(+), 16 deletions(-) create mode 100644 src/migrations/20210129210854-add-fileSize-to-files.js diff --git a/frontend/src/components/FileUploader.js b/frontend/src/components/FileUploader.js index a6a027e66a..0cb3433e8a 100644 --- a/frontend/src/components/FileUploader.js +++ b/frontend/src/components/FileUploader.js @@ -21,7 +21,7 @@ function Dropzone(props) { let attachmentType; if (id === 'attachments') { attachmentType = 'ATTACHMENT'; - } else if (id === 'other-resources') { + } else if (id === 'otherResources') { attachmentType = 'RESOURCE'; } e.forEach(async (file) => { @@ -31,12 +31,12 @@ function Dropzone(props) { data.append('attachmentType', attachmentType); data.append('file', file); await uploadFile(data); + onChange([{originalFileName: file.name, fileSize: file.size, status: "Uploaded"}]) } catch (error) { // eslint-disable-next-line no-console console.log(error); } }); - onChange(e); }; const { getRootProps, getInputProps } = useDropzone({ onDrop }); @@ -87,13 +87,13 @@ const FileTable = ({ onFileRemoved, files }) => { {files.map((file, index) => ( {files.map((file, index) => ( - + @@ -112,8 +116,8 @@ const FileTable = ({ onFileRemoved, files }) => ( aria-label="remove file" onClick={() => { onFileRemoved(index); }} > - - + + diff --git a/src/models/file.js b/src/models/file.js index a105396655..2ab2a43b8a 100644 --- a/src/models/file.js +++ b/src/models/file.js @@ -32,6 +32,7 @@ module.exports = (sequelize, DataTypes) => { type: DataTypes.ENUM('ATTACHMENT', 'RESOURCE'), allowNull: false, }, + // File size in bytes fileSize: { type: DataTypes.INTEGER, allowNull: false, diff --git a/src/routes/files/handlers.js b/src/routes/files/handlers.js index 8927102dd7..7e34e7a513 100644 --- a/src/routes/files/handlers.js +++ b/src/routes/files/handlers.js @@ -69,7 +69,7 @@ export default async function uploadHandler(req, res) { const { path, originalFilename, size } = files.file[0]; const { reportId, attachmentType } = fields; if (!size) { - res.status(400).send({ error: 'improper file size' }); + res.status(400).send({ error: 'fileSize required' }); } if (!reportId) { res.status(400).send({ error: 'reportId required' }); From 95cdef92b66f7e8970ca3837fad2eb8fbaca73a8 Mon Sep 17 00:00:00 2001 From: Chuck McAndrew Date: Tue, 2 Feb 2021 21:31:23 -0500 Subject: [PATCH 62/64] fmt --- frontend/src/components/FileUploader.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/components/FileUploader.js b/frontend/src/components/FileUploader.js index 647d0321b4..6239e4bc19 100644 --- a/frontend/src/components/FileUploader.js +++ b/frontend/src/components/FileUploader.js @@ -35,7 +35,7 @@ function Dropzone(props) { data.append('reportId', reportId); data.append('attachmentType', attachmentType); data.append('file', file); - res = await uploadFile(data); + await uploadFile(data); } catch (error) { setErrorMessage(`${file.name} failed to upload`); // eslint-disable-next-line no-console From df59505b16c2105f958feb2db123b3c1b14a057b Mon Sep 17 00:00:00 2001 From: Chuck McAndrew Date: Wed, 3 Feb 2021 14:07:27 -0500 Subject: [PATCH 63/64] fix review page --- frontend/src/pages/ActivityReport/Pages/reviewItem.js | 3 +++ src/routes/files/handlers.js | 7 +------ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/frontend/src/pages/ActivityReport/Pages/reviewItem.js b/frontend/src/pages/ActivityReport/Pages/reviewItem.js index 29d5c1431f..168c079306 100644 --- a/frontend/src/pages/ActivityReport/Pages/reviewItem.js +++ b/frontend/src/pages/ActivityReport/Pages/reviewItem.js @@ -68,6 +68,9 @@ const Item = ({ label, value, path }) => { if (!Array.isArray(value)) { values = [value]; } + if (label === 'Other Resources' || label === 'Attachments') { + values = values.map((v) => v.originalFileName); + } if (path) { values = values.map((v) => _.get(v, path)); diff --git a/src/routes/files/handlers.js b/src/routes/files/handlers.js index 7e34e7a513..ab98d64736 100644 --- a/src/routes/files/handlers.js +++ b/src/routes/files/handlers.js @@ -17,12 +17,7 @@ const logContext = { }; export const createFileMetaData = async ( - originalFileName, - s3FileName, - reportId, - attachmentType, - fileSize, -) => { + originalFileName, s3FileName, reportId, attachmentType, fileSize) => { const newFile = { activityReportId: reportId, originalFileName, From cf80fa5b77dcc0e4c87d9fd6ec74d8c71bf1b4f1 Mon Sep 17 00:00:00 2001 From: Chuck McAndrew Date: Wed, 3 Feb 2021 15:31:57 -0500 Subject: [PATCH 64/64] add path to attachments --- frontend/src/pages/ActivityReport/Pages/reviewItem.js | 3 --- frontend/src/pages/ActivityReport/Pages/topicsResources.js | 4 ++-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/frontend/src/pages/ActivityReport/Pages/reviewItem.js b/frontend/src/pages/ActivityReport/Pages/reviewItem.js index 168c079306..29d5c1431f 100644 --- a/frontend/src/pages/ActivityReport/Pages/reviewItem.js +++ b/frontend/src/pages/ActivityReport/Pages/reviewItem.js @@ -68,9 +68,6 @@ const Item = ({ label, value, path }) => { if (!Array.isArray(value)) { values = [value]; } - if (label === 'Other Resources' || label === 'Attachments') { - values = values.map((v) => v.originalFileName); - } if (path) { values = values.map((v) => _.get(v, path)); diff --git a/frontend/src/pages/ActivityReport/Pages/topicsResources.js b/frontend/src/pages/ActivityReport/Pages/topicsResources.js index f37f74e543..245406ee99 100644 --- a/frontend/src/pages/ActivityReport/Pages/topicsResources.js +++ b/frontend/src/pages/ActivityReport/Pages/topicsResources.js @@ -97,14 +97,14 @@ const sections = [ anchor: 'resources', items: [ { label: 'Resources used', name: 'resourcesUsed' }, - { label: 'Other resources', name: 'otherResources', path: 'name' }, + { label: 'Other resources', name: 'otherResources', path: 'originalFileName' }, ], }, { title: 'Attachments', anchor: 'attachments', items: [ - { label: 'Attachments', name: 'attachments' }, + { label: 'Attachments', name: 'attachments', path: 'originalFileName' }, ], }, ];
Name + +
{file.name}
- {file.name} + {file.originalFileName} - {`${(file.size / 1000).toFixed(1)} KB`} + {`${(file.fileSize / 1000).toFixed(1)} KB`} - Uploaded + {file.status}
{file.originalFileName}