diff --git a/exampleData/cactus.bed b/exampleData/cactus.bed index 88b6d098..ec4a3dd9 100644 --- a/exampleData/cactus.bed +++ b/exampleData/cactus.bed @@ -1,5 +1,6 @@ ref 10 100 this is a test region +ref 500 600 region without read tracks chunk-cactus-no-reads ref 1000 2000 another region with a very long description to see how it will be displayed ref 2000 3000 pre-fetched region chunk-ref-2000-3000 ref 4000 4500 pre-fetched region but the chunk is missing test_prechunk_missing -ref 482 2450 region without read tracks chunk-cactus-no-reads + diff --git a/exampleData/chunk-ref-2000-3000/chunk.vg b/exampleData/chunk-ref-2000-3000/chunk.vg index 93ed2abd..f4495a65 100644 Binary files a/exampleData/chunk-ref-2000-3000/chunk.vg and b/exampleData/chunk-ref-2000-3000/chunk.vg differ diff --git a/exampleData/chunk-ref-2000-3000/chunk_0_ref_2000_3000.gam b/exampleData/chunk-ref-2000-3000/chunk_0_ref_2000_3000.gam deleted file mode 100644 index 5cc5575a..00000000 Binary files a/exampleData/chunk-ref-2000-3000/chunk_0_ref_2000_3000.gam and /dev/null differ diff --git a/exampleData/chunk-ref-2000-3000/chunk_contents.txt b/exampleData/chunk-ref-2000-3000/chunk_contents.txt index f3779155..ef0b2b44 100644 --- a/exampleData/chunk-ref-2000-3000/chunk_contents.txt +++ b/exampleData/chunk-ref-2000-3000/chunk_contents.txt @@ -1,5 +1,5 @@ chunk.vg -chunk_0_ref_2000_3000.annotate.txt -chunk_0_ref_2000_3000.gam +chunk_0_ref_1955_5023.annotate.txt +chunk_0_ref_1955_5023.gam regions.tsv tracks.json diff --git a/exampleData/chunk-ref-2000-3000/regions.tsv b/exampleData/chunk-ref-2000-3000/regions.tsv index f597b1a9..a6227384 100644 --- a/exampleData/chunk-ref-2000-3000/regions.tsv +++ b/exampleData/chunk-ref-2000-3000/regions.tsv @@ -1 +1 @@ -ref 2000 3000 chunk_0_ref_2000_3000.gam chunk_0_ref_2000_3000.annotate.txt +ref 1955 5024 exampleData/chunk-ref-2000-3000/chunk_0_ref_1955_5023.gam exampleData/chunk-ref-2000-3000/chunk_0_ref_1955_5023.annotate.txt diff --git a/exampleData/chunk-ref-2000-3000/tracks.json b/exampleData/chunk-ref-2000-3000/tracks.json index e186c674..ec87ae80 100644 --- a/exampleData/chunk-ref-2000-3000/tracks.json +++ b/exampleData/chunk-ref-2000-3000/tracks.json @@ -1,10 +1,11 @@ [ { - "trackFile": "exampleData/cactus.vg", + "trackFile": "exampleData/cactus.vg.xg", "trackType": "graph", "trackColorSettings": { - "mainPalette": "plainColors", - "auxPalette": "greys" + "mainPalette": "#000000", + "auxPalette": "greys", + "colorReadsByMappingQuality": false } }, { @@ -12,7 +13,8 @@ "trackType": "read", "trackColorSettings": { "mainPalette": "blues", - "auxPalette": "reds" + "auxPalette": "reds", + "colorReadsByMappingQuality": false } } ] diff --git a/exampleData/chunk-without-source/chunk.vg b/exampleData/chunk-without-source/chunk.vg new file mode 100644 index 00000000..f4495a65 --- /dev/null +++ b/exampleData/chunk-without-source/chunk.vg @@ -0,0 +1,162 @@ +VGô% +g +cAGACGGAGTCTTGCTCTTGTTGCTCAGCCTGGAATGCAATGGCACGATCTCAGCTCACTGCAACCTCCACCTCCCGGGTTCAAGCAATTCTCCTGCCTC +g +cAGCCTCCCAAGTAGCAGGGATTACAGGTGCCTGCCACCATGCCAGGCTAATTGTTTTTTCTTTTTTTTCAGATGGAGTCTCACTCTGTCACTCAGGCTG +g +cGATTGTGATGGTGTGATCTCAGCTCACTGCAACCTCAACATCCTGGGTTCAAGCGATTCTCCTGCCTCAGTCTCCCAAGTAGCTGGGACTACAAGTGCG +g +cTGCCACCATGCCTGGCTAATTTTTTTTAGTATTTTTAGTAGAGATGGGGTTTCGCCATATTGGCCAGGCTGGTCTCAAACTCCTGATGTCAGGTGATCC +g +cGCCCTGAGGCTGAGGCAGGAGAATCATTTAAACCCAGGAGGCGGAGGTTGCAGTGAGCCAAGACTGGGCCACTGCACTCCAGCCTGCTAAGTGACAGAG +g +cTGAGACTCCACCTCAAAAAAAAAAAAAAAAGGCAATGCTTCAGGACATAAGGCCTTGCTCTGAAGAGGCCCTAGGAGTGACTCCTGGTGACAGTGAAAG +g +cCCCACAGCCTCTGGCAACTGTATTAACATGAACTTCAATCTGTTAAAGGAAAGCCACCAGGAAAACAGCACTGTAATTTAACGATGTGGAAAAATGTAT +g +cGTAATATCTTAAGGAAAAAAGCAAAACAGTGTAATTATGATCACATTTTATAAAATACACGTGTATATATACGCACATATGCCTGGTGGAGTTTTATGG! +g +cTGATCATCTCCAAGTGGTGGAATTACTGGGATTATTTTATTGTTTTTGTGTAAATTTATACTTTCTTTTTTCTTTTTGAGACACGGTCTCGCTCTGTCG" +g +cCCCAGGCTGGAGTACAGTGGTGTGATCGTGGCTCACTGAAGCATCAACCTCCTGAGCTCAAGTGATCCTCCCACCTCAGCTTCCCAAGTAGCTGCGACT# +g +cACAGGCATCTGCCACCACACCCAGCTACTTTTTAAATTTGTTGTACAGATGAAGTCTCCTTATGTTGCCCAGGCTGGTCTCGAACTTCTAGGCTCCCAC$ +g +cCTTGACCTCCATCTTGACCTCCCAAAGTGCTGGAATTATAGGCATGAGCCACCATGCCCGGCCTTGATTTATGTTTTTGTGATGAACATTCATATCTTA% +g +cCTCCCACCCCATGGAAACAGTTCATGTATTACTTTTACAATATAAAACAAATAACAATAAAAACATCAAAAAGACATTTTAGCCATTCATTCAACAAAT& +g +cATTTAAAATGTGCCAAGAACTGTGCTACTCAAGCACCAGGTAATGAGTGATAAACCAAACCCATGCAAAAGGACCCCATATAGCACAGGTACATGCAGG' +g +cCACCTTACCATGGAAGCCATTGTCCTCTGTCCAGGCATCTGGCTGCACAACCACAATTGGGTGGACACCCTGGATCCCCAGGAAGGAAAGAGCATTCAA( +g +cAGTGTCAAAGTAGGACTACTGGAACTGTCACTTCATCATTTTTTTTGTTTGTTTTTGAGACAGGGTCTTGCTCTGTCACCCAGGCTGGAGTGCAGTGGT) +g +cGTGATCTCAGCTCACTGCCACCTCTGCCTCCTGGGCTCAAGCAATCCTTCCATCTCAGCCTCCTAAGTAGCTGGAACTACAGACACGTACCACCACCCC* +g +cTGGCTAATTTTTTTGTATTTTTGGTAGAGACAGGGTTTTGCCATGTTGCCCAGGCTGGTCTCAAACTCCTGGGCTCAACTTCACCCCCGGGATTATAGG+ +g +cCATGAGCCACCGCACCCAGCCTTGGCTAATTTTTAATAATTTTTTTGTAGACATGAGGTCCTACTGTATTGCCCAGGCTGGTCTTCAGCTCCCAGGCTC, +g +cAAGCGATTCTCCCACCTTGGCCTCCCAGTGTTGTGATTACAGGGGTGGGGCACTGGCCCAGCCCATCATTTCTCTCTCTCTCTTTTTTTTTGAGACGGA- +g +cGTCTCGCTCTGTCGCCCGGGCTGGAGTGCAGTGGCGCGATCTTGGCTCACTGCAACCTCCGCCTCCGGGGTTCAAGCGATTCTCCTGCCCCAGCCCCTC. +g +cAAGTAGCTGGGACTACAGGCGTGCGCCCCTACGCCCAGCTAATTTTTGTATTTTTAGTAGAGACGGGGTTTCGCCATGTTGGTTGGCCAGGATGGTCTC/ +g +cGATCTCTTGACCTCGTGATCTGCCCACCTCAGCCTCCCAAAGTGCTGGGATTACAGGCGTGAGCCACCGCACCTAGCTTTTCTCTCTCTCTCTTTTTTT0 +g +cTTTTTTTTAGACAAAGTCTCACTCTGTCACCCAGTCTGGAGTGCAGTGGTGCAATCTTGGCTCACTGCAACCTCTGCCTCCCACGTTCAAGCGATGCTC1 +g +cACACCTCAACTTCCCAAATAGCTGGCATTACAGGCATGCTCCACCAGGCCTGGCTACTTTTTGTTTTTTTTTTTTTAGTACAGATGGGGTTTCACCATG2 +g +cTTGGCCAGGCTGGTCTCAAACTCCTGACAAGTGATCCACCTGCCTCGGCCTCCCAAAGTGCTGGGATTACAGACATGAGCCACCATGCCCAGCCTCCAG3 +g +cCCCATCATTTCTTGATGATTTGTTGAAACACAGTATGCTGGGGCAGTCACAGAGAGGAGGGGGAGGGACATATGGGAAAAAGAGTTAGAGGGAAAAAGT4 +g +cCTTCCCTCAGTATATTTAATATGTGCAGTTCTCAAATCCTTACCCATCCCTTACAGATGGAGTCTTTTGGCACAGGTATGTGGGCAGAGAAGACTTCTG5 +g +cAGGCTACAGTAGGGGCATCCATAGGGACTGACAGGTGCCAGTCTTGCTCACAGGAGAGAATATTGTGTCCTCCCTCTCTGACAGGGCACCCAATACTTA6 +g +cCTGTGCCAAGGGTGAATGATGAAAGCTCCTTCACCACAGAAGCACCACACAGCTGTACCATCCATTCCAGTTGATCTAAAATGGACATTTAGATGTAAA7 +g +cATCACTGCAGTAATCTGCATACTTAACCCAGGCCCTCTACCCTACACTCTCCGGATGAAGGCTTATAGCAAGACCTCTCAATGGGAGAGTCTGTCTCTC8  !!""##$$%%&&''(())**++,,--..//00112233445566778„ +GI262359905[76167] +8 cc( +7 cc( +6 cc( +5 cc( +4 cc( +3 cc( +2 cc( +1 cc( +0 cc(  +/ cc( + +. cc(  +- cc(  +, cc(  ++ cc( +* cc( +) cc( +( cc( +' cc( +& cc( +% cc( +$ cc( +# cc( +" cc( +! cc( + cc( + cc( + cc( + cc( + cc( + cc( + cc(Å +GI528476558[1937] +cc( +cc( +cc( +cc( +cc( +cc( + cc( +!cc( +"cc(  +#cc( + +$cc(  +%cc(  +&cc(  +'cc( +(cc( +)cc( +*cc( ++cc( +,cc( +-cc( +.cc( +/cc( +0cc( +1cc( +2cc( +3cc( +4cc( +5cc( +6cc( +7cc( +8cc(· +ref +cc( +cc( +cc( +cc( +cc( +cc( + cc( +!cc( +"cc(  +#cc( + +$cc(  +%cc(  +&cc(  +'cc( +(cc( +)cc( +*cc( ++cc( +,cc( +-cc( +.cc( +/cc( +0cc( +1cc( +2cc( +3cc( +4cc( +5cc( +6cc( +7cc( +8cc( \ No newline at end of file diff --git a/exampleData/chunk-ref-2000-3000/chunk_0_ref_2000_3000.annotate.txt b/exampleData/chunk-without-source/chunk_0_ref_1955_5023.annotate.txt similarity index 100% rename from exampleData/chunk-ref-2000-3000/chunk_0_ref_2000_3000.annotate.txt rename to exampleData/chunk-without-source/chunk_0_ref_1955_5023.annotate.txt diff --git a/exampleData/chunk-without-source/chunk_0_ref_1955_5023.gam b/exampleData/chunk-without-source/chunk_0_ref_1955_5023.gam new file mode 100644 index 00000000..033a0c37 Binary files /dev/null and b/exampleData/chunk-without-source/chunk_0_ref_1955_5023.gam differ diff --git a/exampleData/chunk-without-source/chunk_contents.txt b/exampleData/chunk-without-source/chunk_contents.txt new file mode 100644 index 00000000..ef0b2b44 --- /dev/null +++ b/exampleData/chunk-without-source/chunk_contents.txt @@ -0,0 +1,5 @@ +chunk.vg +chunk_0_ref_1955_5023.annotate.txt +chunk_0_ref_1955_5023.gam +regions.tsv +tracks.json diff --git a/exampleData/chunk-without-source/regions.tsv b/exampleData/chunk-without-source/regions.tsv new file mode 100644 index 00000000..f597b1a9 --- /dev/null +++ b/exampleData/chunk-without-source/regions.tsv @@ -0,0 +1 @@ +ref 2000 3000 chunk_0_ref_2000_3000.gam chunk_0_ref_2000_3000.annotate.txt diff --git a/exampleData/chunk-without-source/tracks.json b/exampleData/chunk-without-source/tracks.json new file mode 100644 index 00000000..8d385cde --- /dev/null +++ b/exampleData/chunk-without-source/tracks.json @@ -0,0 +1,18 @@ +[ + { + "trackFile": "exampleData/nonexistent.vg", + "trackType": "graph", + "trackColorSettings": { + "mainPalette": "plainColors", + "auxPalette": "greys" + } + }, + { + "trackFile": "exampleData/stillnotareal.sorted.gam", + "trackType": "read", + "trackColorSettings": { + "mainPalette": "blues", + "auxPalette": "reds" + } + } +] diff --git a/exampleData/no_source.bed b/exampleData/no_source.bed new file mode 100644 index 00000000..db060a7d --- /dev/null +++ b/exampleData/no_source.bed @@ -0,0 +1 @@ +ref 2000 3000 region with no source graph available chunk-without-source diff --git a/scripts/prepare_chunks.sh b/scripts/prepare_chunks.sh index 028e7804..547f0ae5 100755 --- a/scripts/prepare_chunks.sh +++ b/scripts/prepare_chunks.sh @@ -115,6 +115,10 @@ do printf "$file\n" >> $OUTDIR/chunk_contents.txt done -# Print BED line -cat $OUTDIR/regions.tsv | cut -f1-3 | tr -d "\n" +# Print BED line, using the region we were passed as the coordinates +echo "${REGION%:*}" | tr -d "\n" +printf "\t" +echo "${REGION}" | rev | cut -f1 -d'-' | rev | tr -d "\n" +printf "\t" +echo "${REGION}" | rev | cut -f2 -d'-' | cut -f1 -d':' | rev | tr -d "\n" printf "\t${DESC}\t${OUTDIR}\n" diff --git a/src/server.mjs b/src/server.mjs index 90bfba29..a6fd896f 100644 --- a/src/server.mjs +++ b/src/server.mjs @@ -206,19 +206,23 @@ async function lockDirectories(directoryPaths, lockType, func) { // attempt to acquire a lock for the next directory, and call lockDirectories on the remaining directories const currDirectory = directoryPaths.pop(); return lockDirectory(currDirectory, lockType, async function() { - lockDirectories(directoryPaths, lockType, func); + return lockDirectories(directoryPaths, lockType, func); }) } // runs every hour // deletes any files in the download directory past the set fileExpirationTime set in config -cron.schedule('0 * * * *', () => { +cron.schedule('0 * * * *', async () => { console.log("cron scheduled check"); // attempt to acquire a write lock for each on the directory before attemping to delete files for (const dir of [DOWNLOAD_DATA_PATH, UPLOAD_DATA_PATH]) { - lockDirectory(dir, lockTypes.WRITE_LOCK, async function() { - deleteExpiredFiles(dir); - }); + try { + await lockDirectory(dir, lockTypes.WRITE_LOCK, async function() { + deleteExpiredFiles(dir); + }); + } catch (e) { + console.error("Error checking for expired files in " + dir + ":", e); + } } }); @@ -349,20 +353,28 @@ function getGams(tracks) { return getFilesOfType(tracks, fileTypes.READ); } +// To bridge Express next(err) error handling and async function error +// handling, we have this adapter. It takes Express's next and an async +// function and calls next with any error raised when the async function is +// initially called *or* when its promise is awaited. +async function captureErrors(next, callback) { + try { + await callback(); + } catch (e) { + next(e); + } +} + api.post("/getChunkedData", (req, res, next) => { // We would like this to be an async function, but then Express error // handling doesn't work, because it doesn't detect returned promise // rejections until Express 5. We have to pass an error to next() or else // throw synchronously. - // - // So we set up a promise here and we make sure to handle failures - // ourselves with next(). - - // put readlock on necessary directories while processing chunked data - lockDirectories([DOWNLOAD_DATA_PATH, UPLOAD_DATA_PATH], lockTypes.READ_LOCK, async function() { - let promise = getChunkedData(req, res, next); - promise.catch(next); - await promise; + captureErrors(next, async () => { + // put readlock on necessary directories while processing chunked data + return lockDirectories([DOWNLOAD_DATA_PATH, UPLOAD_DATA_PATH], lockTypes.READ_LOCK, async function() { + return getChunkedData(req, res, next); + }); }); }); @@ -1529,28 +1541,23 @@ async function getChunkTracks (bedFile, chunk) { // Expects a request with a bed file and a chunk name // Returns tracks retrieved from getChunkTracks api.post("/getChunkTracks", (req, res, next) => { - console.log("received request for chunk tracks"); - if (!req.body.bedFile || !req.body.chunk) { - throw new BadRequestError("Invalid request format", req.body.bedFile, req.body.chunk); - } - let promise = (async () => { + captureErrors(next, async () => { + console.log("received request for chunk tracks"); + if (!req.body.bedFile || !req.body.chunk) { + throw new BadRequestError("Invalid request format", req.body.bedFile, req.body.chunk); + } + // tracks are falsy if fetch is unsuccessful // TODO: This operation needs to hold a reader lock on the upload/download directories. // waiting for lock changes to be merged const tracks = await getChunkTracks(req.body.bedFile, req.body.chunk); res.json({ tracks: tracks }); - })(); - - // schedules next to be called if promise is rejected - promise.catch(next); - + }); }); api.post("/getBedRegions", (req, res, next) => { - // Bridge async functions to Express error handling with next(err). Don't - // return a promise. - let promise = (async () => { + captureErrors(next, async () => { console.log("received request for bedRegions"); const result = { bedRegions: [], @@ -1564,8 +1571,7 @@ api.post("/getBedRegions", (req, res, next) => { } else { throw new BadRequestError("No BED file specified"); } - })(); - promise.catch(next); + }); }); // Load up the given BED file by URL or path, and