Skip to content

Commit 6cb9d13

Browse files
NicolasKiefferNicolasKieffer
authoredJan 15, 2025··
Merge pull request #31 from DataSeer/add_version
Add version
·
v2.0.1v2.0.0
2 parents 1b29c2a + 2345491 commit 6cb9d13

23 files changed

+12633
-6161
lines changed
 

‎.eslintrc.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// .eslintrc.js
1+
// File: .eslintrc.js
22
module.exports = {
33
env: {
44
node: true,

‎.husky/commit-msg

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#!/usr/bin/env sh
2+
. "$(dirname -- "$0")/_/husky.sh"
3+
4+
npx --no -- commitlint --edit

‎.husky/post-tag

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/bin/sh
2+
. "$(dirname "$0")/_/husky.sh"
3+
4+
# Extract version from tag
5+
VERSION=$(git describe --tags --abbrev=0)
6+
# Remove 'v' prefix if present
7+
VERSION=${VERSION#v}
8+
9+
# Update package.json version
10+
npm version $VERSION --no-git-tag-version --allow-same-version
11+
12+
# Stage and commit package.json changes
13+
git add package.json
14+
git commit -m "chore: update package.json version to $VERSION" --no-verify

‎.versionrc.json

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
{
2+
"types": [
3+
{ "type": "feat", "section": "Features" },
4+
{ "type": "fix", "section": "Bug Fixes" },
5+
{ "type": "docs", "section": "Documentation" },
6+
{ "type": "style", "section": "Styling" },
7+
{ "type": "refactor", "section": "Code Refactoring" },
8+
{ "type": "perf", "section": "Performance Improvements" },
9+
{ "type": "test", "section": "Tests" },
10+
{ "type": "build", "section": "Build System" },
11+
{ "type": "ci", "section": "Continuous Integration" },
12+
{ "type": "chore", "section": "Chores" }
13+
],
14+
"commitUrlFormat": "https://github.com/DataSeer/snapshot-api/commits/{{hash}}",
15+
"compareUrlFormat": "https://github.com/DataSeer/snapshot-api/compare/{{previousTag}}...{{currentTag}}",
16+
"releaseCommitMessageFormat": "chore(release): {{currentTag}}",
17+
"skip": {
18+
"bump": false,
19+
"changelog": false,
20+
"commit": false,
21+
"tag": false
22+
}
23+
}

‎CHANGELOG.md

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
# Changelog
2+
3+
All notable changes to this project will be documented in this file.
4+
5+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
6+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7+
8+
## [1.0.0] - 2024-01-09
9+
10+
### Features
11+
- PDF document processing integration with GenShare API
12+
- JWT-based authentication system for all routes
13+
- Role-based access control with allow/block lists per route
14+
- User-specific rate limiting with configurable thresholds
15+
- AWS S3 integration for complete request traceability
16+
- Google Sheets integration for summary logging
17+
- Health monitoring for all dependent services (GenShare, GROBID, DataStet)
18+
- Comprehensive logging system with Winston and Morgan
19+
- Script-based user and permissions management
20+
21+
### Security
22+
- JWT authentication required for all routes
23+
- Route-specific access control through permissions system
24+
- Secure token storage and management
25+
- User-specific rate limiting to prevent abuse
26+
- Complete request traceability in S3 storage
27+
28+
### Added
29+
- Command-line tools for user management
30+
- Command-line tools for permission management
31+
- Log analysis utilities
32+
- Health check endpoints for all services
33+
- Automated S3 storage for all processing requests
34+
- Google Sheets integration for process tracking
35+
- Docker support with multi-stage builds
36+
- CI/CD workflows for development and production
37+
38+
### Documentation
39+
- Complete API documentation
40+
- Installation and configuration guides
41+
- Deployment instructions
42+
- Security considerations
43+
- Contributing guidelines
44+
- Script usage examples
45+
46+
[1.0.0]: https://github.com/DataSeer/snapshot-api/releases/tag/v1.0.0

‎Dockerfile

+5-5
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,14 @@ COPY . .
1616

1717
# Copy default files if originals don't exist
1818
RUN if [ ! -f .env ]; then cp .env.default .env || true; fi && \
19-
if [ ! -f conf/users.json ]; then mkdir -p conf && cp conf/users.json.default conf/users.json || true; fi && \
20-
if [ ! -f conf/permissions.json ]; then mkdir -p conf && cp conf/permissions.json.default conf/permissions.json || true; fi && \
19+
if [ ! -f conf/aws.s3.json ]; then mkdir -p conf && cp conf/aws.s3.json.default conf/aws.s3.json || true; fi && \
2120
if [ ! -f conf/datastet.json ]; then mkdir -p conf && cp conf/datastet.json.default conf/datastet.json || true; fi && \
22-
if [ ! -f conf/grobid.json ]; then mkdir -p conf && cp conf/grobid.json.default conf/grobid.json || true; fi && \
2321
if [ ! -f conf/genshare.json ]; then mkdir -p conf && cp conf/genshare.json.default conf/genshare.json || true; fi && \
24-
if [ ! -f conf/aws.s3.json ]; then mkdir -p conf && cp conf/aws.s3.json.default conf/aws.s3.json || true; fi && \
22+
if [ ! -f conf/googleSheets.credentials.json ]; then mkdir -p conf && cp conf/googleSheets.credentials.json.default conf/googleSheets.credentials.json || true; fi && \
2523
if [ ! -f conf/googleSheets.json ]; then mkdir -p conf && cp conf/googleSheets.json.default conf/googleSheets.json || true; fi && \
26-
if [ ! -f conf/googleSheets.credentials.json ]; then mkdir -p conf && cp conf/googleSheets.credentials.json.default conf/googleSheets.credentials.json || true; fi
24+
if [ ! -f conf/grobid.json ]; then mkdir -p conf && cp conf/grobid.json.default conf/grobid.json || true; fi && \
25+
if [ ! -f conf/permissions.json ]; then mkdir -p conf && cp conf/permissions.json.default conf/permissions.json || true; fi && \
26+
if [ ! -f conf/users.json ]; then mkdir -p conf && cp conf/users.json.default conf/users.json || true; fi
2727

2828
# Expose the port your app runs on
2929
EXPOSE 3000

‎README.md

+353-573
Large diffs are not rendered by default.

‎commitlint.config.js

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
//File: commitlint.config.js
2+
module.exports = {
3+
extends: ['@commitlint/config-conventional']
4+
};

‎conf/permissions.json.default

+7
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,13 @@
77
"blocked": []
88
}
99
},
10+
"/versions": {
11+
"GET": {
12+
"description": "Get version of all Snapshot services",
13+
"allowed": ["admin"],
14+
"blocked": []
15+
}
16+
},
1017
"/processPDF": {
1118
"POST": {
1219
"description": "Process a PDF file",

‎package-lock.json

+12,011-5,574
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json

+12-3
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,11 @@
1313
"analyze-logs": "node scripts/analyze_logs.js",
1414
"lint": "eslint . --ext .js",
1515
"lint:fix": "eslint . --ext .js --fix",
16-
"prepare": "husky install"
16+
"prepare": "husky install",
17+
"sync-version": "node scripts/sync_version.js",
18+
"version": "conventional-changelog -p angular -i CHANGELOG.md -s && git add CHANGELOG.md",
19+
"post-version": "git push && git push --tags",
20+
"release": "standard-version"
1721
},
1822
"lint-staged": {
1923
"*.js": "eslint --fix"
@@ -29,15 +33,20 @@
2933
"morgan": "^1.10.0",
3034
"multer": "^1.4.4-lts.1",
3135
"readline": "^1.3.0",
36+
"semver": "^7.6.3",
3237
"uuid": "^8.3.2",
3338
"winston": "^3.15.0"
3439
},
3540
"devDependencies": {
41+
"@commitlint/cli": "^19.6.1",
42+
"@commitlint/config-conventional": "^19.6.0",
43+
"conventional-changelog-cli": "^5.0.0",
3644
"eslint": "^8.56.0",
3745
"eslint-config-prettier": "^9.1.0",
3846
"eslint-plugin-import": "^2.29.1",
3947
"eslint-plugin-node": "^11.1.0",
40-
"husky": "^8.0.0",
41-
"lint-staged": "^15.0.0"
48+
"husky": "^8.0.3",
49+
"lint-staged": "^15.0.0",
50+
"standard-version": "^9.5.0"
4251
}
4352
}

‎scripts/analyze_logs.js

+1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
// File: scripts/manage_permissions.js
12
const fs = require('fs');
23
const readline = require('readline');
34
const path = require('path');

‎scripts/manage_permissions.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// scripts/manage_permissions.js
1+
// File: scripts/manage_permissions.js
22
const fs = require('fs');
33
const path = require('path');
44

‎scripts/sync_version.js

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
// File: scripts/sync_version.js
2+
const fs = require('fs');
3+
const { execSync } = require('child_process');
4+
const semver = require('semver');
5+
6+
function getCurrentVersion() {
7+
// Get the latest tag from git
8+
try {
9+
const tag = execSync('git describe --tags --abbrev=0').toString().trim();
10+
// Remove 'v' prefix if present
11+
return tag.startsWith('v') ? tag.slice(1) : tag;
12+
} catch (error) {
13+
console.log('No git tags found, using package.json version');
14+
const pkg = JSON.parse(fs.readFileSync('./package.json'));
15+
return pkg.version;
16+
}
17+
}
18+
19+
function updatePackageVersion(version) {
20+
const packagePath = './package.json';
21+
const pkg = JSON.parse(fs.readFileSync(packagePath));
22+
23+
// Only update if the new version is greater
24+
if (semver.gt(version, pkg.version)) {
25+
pkg.version = version;
26+
fs.writeFileSync(packagePath, JSON.stringify(pkg, null, 2) + '\n');
27+
console.log(`Updated package.json version to ${version}`);
28+
} else {
29+
console.log('Package version is already up to date');
30+
}
31+
}
32+
33+
// Main execution
34+
const version = getCurrentVersion();
35+
updatePackageVersion(version);

‎src/controllers/genshareController.js

+6
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
// File: src/controllers/genshareController.js
2+
const packageJson = require('../../package.json');
23
const fs = require('fs');
34
const axios = require('axios');
45
const FormData = require('form-data');
@@ -79,6 +80,8 @@ const appendToSummary = async ({ session, errorStatus, req }) => {
7980
// Log to Google Sheets
8081
await appendToSheet([
8182
`=HYPERLINK("${session.url}","${session.requestId}")`, // Query ID with S3 link
83+
session.getSnapshotAPIVersion(), // Snapshot API version
84+
session.getGenshareVersion(), // GenShare version
8285
errorStatus, // Error status
8386
convertToGoogleSheetsDate(now), // Date
8487
convertToGoogleSheetsTime(now), // Time
@@ -120,6 +123,7 @@ module.exports.getGenShareHealth = async (req, res) => {
120123
module.exports.processPDF = async (req, res) => {
121124
// Initialize processing session
122125
const session = new ProcessingSession(req.user.id, req.file);
126+
session.setSnapshotAPIVersion(`v${packageJson.version}`);
123127
let errorStatus = "No"; // Initialize error status
124128

125129
try {
@@ -229,6 +233,8 @@ module.exports.processPDF = async (req, res) => {
229233
filteredData = { response: response.data.response };
230234
}
231235

236+
session.setGenshareVersion(`v${response.data.version}`);
237+
232238
// Save session data and clean up
233239
session.addLog('Response processing completed');
234240
await session.saveToS3();

‎src/controllers/versionController.js

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
// File: src/controllers/versionController.js
2+
const packageJson = require('../../package.json');
3+
4+
/**
5+
* Get current version information
6+
* @param {Request} req - Express request object
7+
* @param {Response} res - Express response object
8+
*/
9+
const getVersions = async (req, res) => {
10+
try {
11+
res.json({
12+
"snapshot-api": `v${packageJson.version}`
13+
});
14+
} catch (error) {
15+
console.error('Error getting version info:', error);
16+
res.status(500).send('Error retrieving version information');
17+
}
18+
};
19+
20+
module.exports = {
21+
getVersions
22+
};

‎src/middleware/permissions.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// middleware/permissions.js
1+
// File: src/middleware/permissions.js
22
const { getPermissions } = require('../utils/permissionsManager');
33

44
const normalizeUrl = (url) => {

‎src/routes/index.js

+2
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ const { getGrobidHealth } = require('../controllers/grobidController');
66
const { getDatastetHealth } = require('../controllers/datastetController');
77
const { getPing } = require('../controllers/healthController');
88
const { getApiRoutes } = require('../controllers/apiController');
9+
const { getVersions } = require('../controllers/versionController');
910
const rateLimiter = require('../utils/rateLimiter');
1011

1112
const router = express.Router();
@@ -14,6 +15,7 @@ const router = express.Router();
1415
const upload = multer({ dest: 'tmp/' });
1516

1617
router.get('/', rateLimiter, getApiRoutes);
18+
router.get('/versions', rateLimiter, getVersions);
1719
router.post('/processPDF', rateLimiter, upload.single('file'), processPDF);
1820

1921
// Health check endpoints

‎src/utils/logger.js

-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
// File: src/utils/logger.js
2-
32
const winston = require('winston');
43
const morgan = require('morgan');
54

‎src/utils/permissionsManager.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// utils/permissionsManager.js
1+
// File: src/utils/permissionsManager.js
22
const fs = require('fs');
33
const config = require('../config');
44

‎src/utils/s3Storage.js

+35-1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ const AWS = require('aws-sdk');
33
const crypto = require('crypto');
44
const fs = require('fs');
55

6+
const { isValidVersion } = require('./version');
7+
68
// Load S3 configuration from JSON file
79
// eslint-disable-next-line node/no-unpublished-require
810
const s3Config = require('../../conf/aws.s3.json');
@@ -71,6 +73,8 @@ class ProcessingSession {
7173
this.startTime = new Date();
7274
this.endTime = null;
7375
this.duration = -1;
76+
this.snapshotAPIVersion = "";
77+
this.genshareVersion = "";
7478

7579
// Add initial log with session start
7680
this.addLog('Session started', 'INFO');
@@ -79,6 +83,34 @@ class ProcessingSession {
7983
}
8084
}
8185

86+
getSnapshotAPIVersion() {
87+
return this.snapshotAPIVersion;
88+
}
89+
90+
getGenshareVersion() {
91+
return this.genshareVersion;
92+
}
93+
94+
setSnapshotAPIVersion(version) {
95+
if (!isValidVersion(version)) {
96+
this.snapshotAPIVersion = '';
97+
this.addLog(`Invalid Snapshot API Version format: ${version}. Setting empty string.`, 'WARN');
98+
return;
99+
}
100+
this.snapshotAPIVersion = version;
101+
this.addLog(`Snapshot API Version set to: ${version}`, 'INFO');
102+
}
103+
104+
setGenshareVersion(version) {
105+
if (!isValidVersion(version)) {
106+
this.genshareVersion = '';
107+
this.addLog(`Invalid Genshare Version format: ${version}. Setting empty string.`, 'WARN');
108+
return;
109+
}
110+
this.genshareVersion = version;
111+
this.addLog(`Genshare Version set to: ${version}`, 'INFO');
112+
}
113+
82114
getBasePath() {
83115
return `${s3Config.s3Folder}/${this.userId}/${this.requestId}`;
84116
}
@@ -136,7 +168,9 @@ class ProcessingSession {
136168
startDate: formatLogDate(this.startTime),
137169
endDate: formatLogDate(this.endTime),
138170
duration: `${this.duration}ms`,
139-
hasFile: !!this.file
171+
hasFile: !!this.file,
172+
snapshotAPIVersion: this.snapshotAPIVersion,
173+
genshareVersion: this.genshareVersion
140174
};
141175

142176
// Add common files that don't depend on this.file

‎src/utils/tokenManager.js

+1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
// File: src/utils/tokenManager.js
12
const fs = require('fs');
23
const config = require('../config');
34

‎src/utils/version.js

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
// File: src/utils/version.js
2+
const semver = require('semver');
3+
4+
const VERSION_REGEX = /^v\d+\.\d+\.\d+$/;
5+
6+
/**
7+
* Validates a version string
8+
* @param {string} version - Version string to validate (with 'v' prefix)
9+
* @returns {boolean} - Whether the version is valid
10+
*/
11+
const isValidVersion = (version) => {
12+
if (!VERSION_REGEX.test(version)) return false;
13+
return semver.valid(version.substring(1)) !== null;
14+
};
15+
16+
/**
17+
* Normalizes a version string by ensuring it has a 'v' prefix
18+
* @param {string} version - Version string to normalize
19+
* @returns {string} - Normalized version string
20+
*/
21+
const normalizeVersion = (version) => {
22+
if (!version) return '';
23+
version = version.trim();
24+
if (!version.startsWith('v')) {
25+
version = `v${version}`;
26+
}
27+
return isValidVersion(version) ? version : '';
28+
};
29+
30+
/**
31+
* Compares two versions
32+
* @param {string} v1 - First version
33+
* @param {string} v2 - Second version
34+
* @returns {number} - -1 if v1 < v2, 0 if v1 = v2, 1 if v1 > v2
35+
*/
36+
const compareVersions = (v1, v2) => {
37+
if (!isValidVersion(v1) || !isValidVersion(v2)) return 0;
38+
return semver.compare(
39+
v1.substring(1),
40+
v2.substring(1)
41+
);
42+
};
43+
44+
module.exports = {
45+
isValidVersion,
46+
normalizeVersion,
47+
compareVersions
48+
};

0 commit comments

Comments
 (0)
Please sign in to comment.