From 80fa5cc58c5c2d2eb9479144fef11b848e2b7260 Mon Sep 17 00:00:00 2001 From: Thomas Thelen Date: Tue, 23 Feb 2021 14:52:35 -0800 Subject: [PATCH 01/45] Replace the 0.98 download with speedbagit --- .../nceas/metacat/dataone/MNodeService.java | 196 ++++++++++-------- .../restservice/v1/MNResourceHandler.java | 10 +- .../restservice/v2/MNResourceHandler.java | 67 +++--- .../util/DeleteOnCloseFileInputStream.java | 51 ----- 4 files changed, 144 insertions(+), 180 deletions(-) delete mode 100644 src/edu/ucsb/nceas/metacat/util/DeleteOnCloseFileInputStream.java diff --git a/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java b/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java index 6c1567c85..11f245e3d 100644 --- a/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java +++ b/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java @@ -38,6 +38,7 @@ import java.math.BigInteger; import java.net.URISyntaxException; import java.nio.charset.Charset; +import java.nio.file.Paths; import java.security.NoSuchAlgorithmException; import java.sql.SQLException; import java.util.ArrayList; @@ -138,6 +139,7 @@ import org.dataone.service.types.v1_1.QueryField; import org.dataone.service.util.Constants; import org.dataone.service.util.TypeMarshaller; +import org.dataone.speedbagit.SpeedBagIt; import org.dspace.foresite.OREException; import org.dspace.foresite.OREParserException; import org.dspace.foresite.ORESerialiserException; @@ -717,6 +719,21 @@ protected void removeIdFromIdentifierTable(Identifier id){ } } + + public Identifier create(Session session,, InputStream object) throws InvalidToken, ServiceFailure, NotAuthorized, + IdentifierNotUnique, UnsupportedType, InsufficientResources, InvalidSystemMetadata, NotImplemented, InvalidRequest { + + if (isReadOnlyMode()) { + throw new ServiceFailure("1190", ReadOnlyChecker.DATAONEERROR); + } + // check for null session + if (session == null) { + throw new InvalidToken("1110", "The session in the request was null. Ensure that your credentials " + + "were included in the request."); + } + + } + public Identifier create(Session session, Identifier pid, InputStream object, SystemMetadata sysmeta) throws InvalidToken, ServiceFailure, NotAuthorized, IdentifierNotUnique, UnsupportedType, InsufficientResources, InvalidSystemMetadata, NotImplemented, InvalidRequest { @@ -2527,28 +2544,50 @@ private List lookupOreFor(Session session, Identifier guid) { return retList; } + /** + * Exports a data package to disk using the BagIt 0.97 specification. + * + * The ultimate goal is, for every file being added to the bag, + * get an InputStream to it and add it to the SpeedBagIt instance. + * + * During the export process, a PDF file is generated and included in the bag. + * The conversion works by using an XSLT with the EML to transform it into HTML, + * which is then converted to a PDF file. During this process the HTML and CSS file are + * written to disk, read in by the HtmlToPdf object, and then written back to disk. + * + * The PDF file and pid-mapping.txt files are both stored in memory throughout the + * export process. + * + * @param session Information about the user performing the request + * @param formatId + * @param pid The pid of the resource map + * @return A stream of a bag + * @throws InvalidToken + * @throws ServiceFailure + * @throws NotAuthorized + * @throws InvalidRequest + * @throws NotImplemented + * @throws NotFound + */ @Override public InputStream getPackage(Session session, ObjectFormatIdentifier formatId, Identifier pid) throws InvalidToken, ServiceFailure, NotAuthorized, InvalidRequest, NotImplemented, NotFound { if(formatId == null) { - throw new InvalidRequest("2873", "The format type can't be null in the getpackage method."); + throw new InvalidRequest("2873", "The format id wasn't specified in the request. " + + "Ensure that the format id is properly set in the request."); } else if(!formatId.getValue().equals("application/bagit-097")) { - throw new NotImplemented("", "The format "+formatId.getValue()+" is not supported in the getpackage method"); + throw new NotImplemented("", "The format "+formatId.getValue()+" is not a supported format."); } String serviceFailureCode = "2871"; Identifier sid = getPIDForSID(pid, serviceFailureCode); if(sid != null) { pid = sid; } - InputStream bagInputStream = null; - BagFactory bagFactory = new BagFactory(); - Bag bag = bagFactory.createBag(); - - // track the temp files we use so we can delete them when finished - List tempFiles = new ArrayList(); - - // the pids to include in the package + // Create a bag that is version 0.97 and has tag files that contain MD5 checksums + SpeedBagIt speedBag = new SpeedBagIt("0.97", "MD5"); + + // The pids of all of the objects in the package List packagePids = new ArrayList(); // catch non-D1 service errors and throw as ServiceFailures @@ -2575,12 +2614,12 @@ public InputStream getPackage(Session session, ObjectFormatIdentifier formatId, //Get the system metadata for this metadata object SystemMetadata metadataSysMeta = this.getSystemMetadata(session, metadataID); - // include user-friendly metadata + // If it's supported metadata, create the PDF file out of it if (ObjectFormatCache.getInstance().getFormat(metadataSysMeta.getFormatId()).getFormatType().equals("METADATA")) { InputStream metadataStream = this.get(session, metadataID); try { - // transform + // Set the properties for the XSLT transform String format = "default"; DBTransform transformer = new DBTransform(); @@ -2614,42 +2653,56 @@ public InputStream getPackage(Session session, ObjectFormatIdentifier formatId, // finally, get the HTML back ContentTypeByteArrayInputStream resultInputStream = new ContentTypeByteArrayInputStream(baos.toByteArray()); - - // write to temp file with correct css path + + // Create a temporary directory to store the html + css. This is required for HtmlToPdf File tmpDir = File.createTempFile("package_", "_dir"); tmpDir.delete(); tmpDir.mkdir(); - File htmlFile = File.createTempFile("metadata", ".html", tmpDir); + + // Create the directory for the CSS. This is required for HtmlToPdf File cssDir = new File(tmpDir, format); cssDir.mkdir(); File cssFile = new File(tmpDir, format + "/" + format + ".css"); - String pdfFileName = metadataID.getValue().replaceAll("[^a-zA-Z0-9\\-\\.]", "_") + "-METADATA.pdf"; - File pdfFile = new File(tmpDir, pdfFileName); - //File pdfFile = File.createTempFile("metadata", ".pdf", tmpDir); - - // put the CSS file in place for the html to find it - String originalCssPath = SystemUtil.getContextDir() + "/style/skins/" + format + "/" + format + ".css"; - IOUtils.copy(new FileInputStream(originalCssPath), new FileOutputStream(cssFile)); - + + // Write the CSS to the file + String originalCssPath = SystemUtil.getContextDir() + "/style/skins/" + format + "/" + format + ".css"; + IOUtils.copy(new FileInputStream(originalCssPath), new FileOutputStream(cssFile)); + + // Create the pdf File that HtmlToPdf will write to + String pdfFileName = metadataID.getValue().replaceAll("[^a-zA-Z0-9\\-\\.]", "_") + "-METADATA.pdf"; + File pdfFile = new File(tmpDir, pdfFileName); + File pdfFile = File.createTempFile("metadata", ".pdf", tmpDir); + // write the HTML file + File htmlFile = File.createTempFile("metadata", ".html", tmpDir); IOUtils.copy(resultInputStream, new FileOutputStream(htmlFile)); // convert to PDF HtmlToPdf.export(htmlFile.getAbsolutePath(), pdfFile.getAbsolutePath()); - - //add to the package - bag.addFileToPayload(pdfFile); - pidMapping.append(metadataID.getValue() + " (pdf)" + "\t" + "data/" + pdfFile.getName() + "\n"); - - // mark for clean up after we are done - htmlFile.delete(); - cssFile.delete(); - cssDir.delete(); - tempFiles.add(tmpDir); - tempFiles.add(pdfFile); // delete this first later on - + + // Now that the PDF file is generated and written to disk, + // delete the HTML & CSS files + htmlFile.delete(); + cssFile.delete(); + cssDir.delete(); + + // Load the PDF file into memory so that it can be deleted from disk + byte[] pdfMetadata = Files.readAllBytes(pdfFile.getAbsolutePath()); + InputStream pdfInputStream = new ByteArrayInputStream(pdfMetadata); + + // Now that the PDF file has been loaded into memory, delete it from disk + pdfFile.delete(); + tmpDir.delete(); + + // Add the pdf to the bag + speedBag.addFile(pdfInputStream, Paths.get("data", pdfFileName); + + // Create a record in the pid mapping file + pidMapping.append(metadataID.getValue() + " (pdf)" + "\t" + "data/" + pdfFile.getName() + "\n"); + } catch (Exception e) { - logMetacat.warn("Could not transform metadata", e); + logMetacat.warn("There was an error generating the PDF file during a package export. " + + "Ensure that the package metadata is valid and supported.", e); } } @@ -2708,18 +2761,15 @@ public InputStream getPackage(Session session, ObjectFormatIdentifier formatId, // just the lone pid in this package //packagePids.add(pid); //throw an invalid request exception - throw new InvalidRequest("2873", "The given pid "+pid.getValue()+" is not a package id (resource map id). Please use a package id instead."); + throw new InvalidRequest("2873", "The given pid "+pid.getValue()+" is not a package " + + "id (resource map id). Please use a package id instead."); } - - //Create a temp file, then delete it and make a directory with that name - File tempDir = File.createTempFile("temp", Long.toString(System.nanoTime())); - tempDir.delete(); - tempDir = new File(tempDir.getPath() + "_dir"); - tempDir.mkdir(); - tempFiles.add(tempDir); - File pidMappingFile = new File(tempDir, "pid-mapping.txt"); - - // loop through the package contents + + /** + * Up to this point, the only file that has been added to the bag is the metadata pdf file. + * The next step is looping over each object in the package, determining its filename, + * getting an InputStream to it, and adding it to the bag. + */ for (Identifier entryPid: packagePids) { //Get the system metadata for each item SystemMetadata entrySysMeta = this.getSystemMetadata(session, entryPid); @@ -2745,44 +2795,15 @@ public InputStream getPackage(Session session, ObjectFormatIdentifier formatId, fileName = entrySysMeta.getFileName().replaceAll("[^a-zA-Z0-9\\-\\.]", "_"); } - //Create a new file for this item and add to the list - File tempFile = new File(tempDir, fileName); - tempFiles.add(tempFile); - - InputStream entryInputStream = this.get(session, entryPid); - IOUtils.copy(entryInputStream, new FileOutputStream(tempFile)); - bag.addFileToPayload(tempFile); + // Add the stream of the file to the bag object & write to the pid mapping file + InputStream entryInputStream = this.get(session, entryPid); + speedBag.addFile(entryInputStream, Paths.get("data", filename)); pidMapping.append(entryPid.getValue() + "\t" + "data/" + tempFile.getName() + "\n"); } - //add the the pid to data file map - IOUtils.write(pidMapping.toString(), new FileOutputStream(pidMappingFile)); - bag.addFileAsTag(pidMappingFile); - tempFiles.add(pidMappingFile); - - bag = bag.makeComplete(); - - ///Now create the zip file - //Use the pid as the file name prefix, replacing all non-word characters - String zipName = pid.getValue().replaceAll("\\W", "_"); - - File bagFile = new File(tempDir, zipName+".zip"); - - bag.setFile(bagFile); - ZipWriter zipWriter = new ZipWriter(bagFactory); - bag.write(zipWriter, bagFile); - bagFile = bag.getFile(); - // use custom FIS that will delete the file when closed - bagInputStream = new DeleteOnCloseFileInputStream(bagFile); - // also mark for deletion on shutdown in case the stream is never closed - bagFile.deleteOnExit(); - tempFiles.add(bagFile); - - // clean up other temp files - for (int i=tempFiles.size()-1; i>=0; i--){ - tempFiles.get(i).delete(); - } - + // Get a stream to the pid mapping file and add it as a tag file, in the bag root + ByteArrayInputStream pidFile = ByteArrayInputStream(pidMapping.toString().getBytes(StandardCharsets.UTF_8)); + speedBag.addFile(pidFile, "pid-mapping.txt", true); } catch (IOException e) { // report as service failure e.printStackTrace(); @@ -2804,14 +2825,17 @@ public InputStream getPackage(Session session, ObjectFormatIdentifier formatId, } catch (OREParserException e) { // report as service failure e.printStackTrace(); - ServiceFailure sf = new ServiceFailure("1030", e.getMessage()); + ServiceFailure sf = new ServiceFailure("1030", "There was an " + + "error while processing the resource map. Ensure that the resource map " + + "for the package is valid. " + e.getMessage()); sf.initCause(e); throw sf; } - - return bagInputStream; + + // Now that all of the files have been added to speedBag, return the InputStream + return speedBag; } - + /** * Archives an object, where the object is either a * data object or a science metadata object. diff --git a/src/edu/ucsb/nceas/metacat/restservice/v1/MNResourceHandler.java b/src/edu/ucsb/nceas/metacat/restservice/v1/MNResourceHandler.java index 5f3f186b6..57704bff6 100644 --- a/src/edu/ucsb/nceas/metacat/restservice/v1/MNResourceHandler.java +++ b/src/edu/ucsb/nceas/metacat/restservice/v1/MNResourceHandler.java @@ -1237,13 +1237,9 @@ protected void getPackage(String pid) throws InvalidToken, ServiceFailure, NotAu id.setValue(pid); InputStream is = MNodeService.getInstance(request).getPackage(session, null, id); - // use the provided filename - String filename = null; - if (is instanceof DeleteOnCloseFileInputStream) { - filename = ((DeleteOnCloseFileInputStream)is).getFile().getName(); - } else { - filename = "dataPackage-" + System.currentTimeMillis() + ".zip"; - } + //Use the pid as the file name prefix, replacing all non-word characters + String filename = pid.getValue().replaceAll("\\W", "_"); + response.setHeader("Content-Disposition", "inline; filename=\"" + filename+"\""); response.setContentType("application/zip"); response.setStatus(200); diff --git a/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java b/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java index c5030dda5..90b78e0ca 100644 --- a/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java +++ b/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java @@ -100,7 +100,6 @@ import edu.ucsb.nceas.metacat.restservice.multipart.DetailedFileInputStream; import edu.ucsb.nceas.metacat.restservice.multipart.MultipartRequestWithSysmeta; import edu.ucsb.nceas.metacat.restservice.multipart.StreamingMultipartRequestResolver; -import edu.ucsb.nceas.metacat.util.DeleteOnCloseFileInputStream; import edu.ucsb.nceas.utilities.PropertyNotFoundException; /** @@ -1504,7 +1503,8 @@ else if(name.equals("count") && value != null) /** * Retrieve data package as Bagit zip - * @param pid + * @param format + * @param pid The pid of the resource map defining the pacakage * @throws NotImplemented * @throws NotFound * @throws NotAuthorized @@ -1514,40 +1514,35 @@ else if(name.equals("count") && value != null) * @throws InvalidRequest */ protected void getPackage(String format, String pid) throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented, IOException, InvalidRequest { - long start = System.currentTimeMillis(); - Identifier id = new Identifier(); - id.setValue(pid); - ObjectFormatIdentifier formatId = null; - if (format != null) { - formatId = new ObjectFormatIdentifier(); - formatId.setValue(format); - } - InputStream is = null; - try { - is = MNodeService.getInstance(request).getPackage(session, formatId , id); - - // use the provided filename - String filename = null; - if (is instanceof DeleteOnCloseFileInputStream) { - filename = ((DeleteOnCloseFileInputStream)is).getFile().getName(); - } else { - filename = "dataPackage-" + System.currentTimeMillis() + ".zip"; - } - response.setHeader("Content-Disposition", "inline; filename=\"" + filename+"\""); - response.setContentType("application/zip"); - response.setStatus(200); - OutputStream out = response.getOutputStream(); - - // write it to the output stream - IOUtils.copyLarge(is, out); - IOUtils.closeQuietly(out); - long end = System.currentTimeMillis(); - logMetacat.info(Settings.PERFORMANCELOG + pid + Settings.PERFORMANCELOG_GET_PACKAGE_METHOD + " Total getPackage method" + Settings.PERFORMANCELOG_DURATION + (end-start)/1000); - - } finally { - IOUtils.closeQuietly(is); - } - } + long start = System.currentTimeMillis(); + Identifier id = new Identifier(); + id.setValue(pid); + ObjectFormatIdentifier formatId = null; + if (format != null) { + formatId = new ObjectFormatIdentifier(); + formatId.setValue(format); + } + InputStream is = null; + try { + is = MNodeService.getInstance(request).getPackage(session, formatId, id); + + //Use the pid as the file name prefix, replacing all non-word characters + String filename = pid.replaceAll("\\W", "_") + ".zip"; + + response.setHeader("Content-Disposition", "inline; filename=\"" + filename + "\""); + response.setContentType("application/zip"); + response.setStatus(200); + OutputStream out = response.getOutputStream(); + + // write it to the output stream + IOUtils.copyLarge(is, out); + IOUtils.closeQuietly(out); + long end = System.currentTimeMillis(); + logMetacat.info(Settings.PERFORMANCELOG + pid + Settings.PERFORMANCELOG_GET_PACKAGE_METHOD + " Total getPackage method" + Settings.PERFORMANCELOG_DURATION + (end - start) / 1000); + } finally { + IOUtils.closeQuietly(is); + } + } protected void publish(String pid) throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented, IOException, diff --git a/src/edu/ucsb/nceas/metacat/util/DeleteOnCloseFileInputStream.java b/src/edu/ucsb/nceas/metacat/util/DeleteOnCloseFileInputStream.java deleted file mode 100644 index a064329ad..000000000 --- a/src/edu/ucsb/nceas/metacat/util/DeleteOnCloseFileInputStream.java +++ /dev/null @@ -1,51 +0,0 @@ -package edu.ucsb.nceas.metacat.util; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; - -/** - * Extension of FileInputSteam that deletes the sourcefile when the - * InputStream is closed. Should typically be used with temporary files - * when a more immediate deletion should be performed than is offered - * by the File.deleteOnExit() method. - * @author leinfelder - * - */ -public class DeleteOnCloseFileInputStream extends FileInputStream { - private File file; - - public DeleteOnCloseFileInputStream(String name) throws FileNotFoundException { - this(new File(name)); - } - - public DeleteOnCloseFileInputStream(File file) throws FileNotFoundException { - super(file); - this.file = file; - } - - /** - * Allow access to the underlying file - careful! - * @return - */ - public File getFile() { - return file; - } - - - /** - * Delete the file when the InputStream is closed - */ - public void close() throws IOException { - try { - super.close(); - } finally { - if (file != null) { - file.delete(); - file = null; - } - } - } - -} From f3cbdcf933ee7d52168fa3e9610e3310e891bd40 Mon Sep 17 00:00:00 2001 From: Thomas Thelen Date: Tue, 23 Feb 2021 16:37:53 -0800 Subject: [PATCH 02/45] Remove LoC BagIt dependancy and add speedbagit --- pom.xml | 35 ++++++----------------------------- 1 file changed, 6 insertions(+), 29 deletions(-) diff --git a/pom.xml b/pom.xml index c1533595e..43707b299 100644 --- a/pom.xml +++ b/pom.xml @@ -64,35 +64,7 @@ datamanager 1.0.0 - - gov.loc - bagit - 4.4 - jar - - - xml-apis - xml-apis - - - org.apache.httpcomponents - httpclient - - - org.apache.httpcomponents - httpmime - - - commons-codec - commons-codec - - - log4j - log4j - - - - + d1_portal org.dataone ${d1_portal_version} @@ -467,5 +439,10 @@ + + org.dataone + speedbagit + 1.0-SNAPSHOT + From 17da9c86c71666e705d08ecd6411a728611397dd Mon Sep 17 00:00:00 2001 From: Thomas Thelen Date: Tue, 23 Feb 2021 16:58:55 -0800 Subject: [PATCH 03/45] Remove LoC BagIt dependancy and add speedbagit --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 43707b299..97d1ade10 100644 --- a/pom.xml +++ b/pom.xml @@ -64,7 +64,7 @@ datamanager 1.0.0 - d1_portal org.dataone ${d1_portal_version} From a1dedbab57bc23d824904a9822e47e022067f0af Mon Sep 17 00:00:00 2001 From: Thomas Thelen Date: Tue, 23 Feb 2021 17:04:28 -0800 Subject: [PATCH 04/45] Fix missing ) --- src/edu/ucsb/nceas/metacat/dataone/MNodeService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java b/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java index 11f245e3d..6d7db8a5f 100644 --- a/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java +++ b/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java @@ -2695,7 +2695,7 @@ public InputStream getPackage(Session session, ObjectFormatIdentifier formatId, tmpDir.delete(); // Add the pdf to the bag - speedBag.addFile(pdfInputStream, Paths.get("data", pdfFileName); + speedBag.addFile(pdfInputStream, Paths.get("data", pdfFileName)); // Create a record in the pid mapping file pidMapping.append(metadataID.getValue() + " (pdf)" + "\t" + "data/" + pdfFile.getName() + "\n"); From f68460dbd4e84ba9003b3e309ee2a4f32d2e7fb8 Mon Sep 17 00:00:00 2001 From: Thomas Thelen Date: Wed, 24 Feb 2021 10:24:14 -0800 Subject: [PATCH 05/45] Remove erroneous create code --- .../ucsb/nceas/metacat/dataone/MNodeService.java | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java b/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java index 6d7db8a5f..2c5410ca6 100644 --- a/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java +++ b/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java @@ -719,21 +719,6 @@ protected void removeIdFromIdentifierTable(Identifier id){ } } - - public Identifier create(Session session,, InputStream object) throws InvalidToken, ServiceFailure, NotAuthorized, - IdentifierNotUnique, UnsupportedType, InsufficientResources, InvalidSystemMetadata, NotImplemented, InvalidRequest { - - if (isReadOnlyMode()) { - throw new ServiceFailure("1190", ReadOnlyChecker.DATAONEERROR); - } - // check for null session - if (session == null) { - throw new InvalidToken("1110", "The session in the request was null. Ensure that your credentials " + - "were included in the request."); - } - - } - public Identifier create(Session session, Identifier pid, InputStream object, SystemMetadata sysmeta) throws InvalidToken, ServiceFailure, NotAuthorized, IdentifierNotUnique, UnsupportedType, InsufficientResources, InvalidSystemMetadata, NotImplemented, InvalidRequest { From 1e1278c53ee868376a69867fbb47078fd9870e63 Mon Sep 17 00:00:00 2001 From: Thomas Thelen Date: Wed, 24 Feb 2021 10:27:57 -0800 Subject: [PATCH 06/45] Remove unused imports --- src/edu/ucsb/nceas/metacat/dataone/MNodeService.java | 1 - src/edu/ucsb/nceas/metacat/restservice/v1/MNResourceHandler.java | 1 - 2 files changed, 2 deletions(-) diff --git a/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java b/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java index 2c5410ca6..42df09ba0 100644 --- a/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java +++ b/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java @@ -169,7 +169,6 @@ import edu.ucsb.nceas.metacat.index.MetacatSolrIndex; import edu.ucsb.nceas.metacat.properties.PropertyService; import edu.ucsb.nceas.metacat.shared.MetacatUtilException; -import edu.ucsb.nceas.metacat.util.DeleteOnCloseFileInputStream; import edu.ucsb.nceas.metacat.util.DocumentUtil; import edu.ucsb.nceas.metacat.util.SkinUtil; import edu.ucsb.nceas.metacat.util.SystemUtil; diff --git a/src/edu/ucsb/nceas/metacat/restservice/v1/MNResourceHandler.java b/src/edu/ucsb/nceas/metacat/restservice/v1/MNResourceHandler.java index 57704bff6..1b569474d 100644 --- a/src/edu/ucsb/nceas/metacat/restservice/v1/MNResourceHandler.java +++ b/src/edu/ucsb/nceas/metacat/restservice/v1/MNResourceHandler.java @@ -88,7 +88,6 @@ import edu.ucsb.nceas.metacat.restservice.multipart.DetailedFileInputStream; import edu.ucsb.nceas.metacat.restservice.multipart.MultipartRequestWithSysmeta; import edu.ucsb.nceas.metacat.restservice.multipart.StreamingMultipartRequestResolver; -import edu.ucsb.nceas.metacat.util.DeleteOnCloseFileInputStream; import edu.ucsb.nceas.utilities.PropertyNotFoundException; import edu.ucsb.nceas.metacat.MetaCatServlet; import edu.ucsb.nceas.metacat.ReadOnlyChecker; From 093cba36ce17a72b2479f3a0826bbec94cb0693f Mon Sep 17 00:00:00 2001 From: Thomas Thelen Date: Wed, 24 Feb 2021 11:29:08 -0800 Subject: [PATCH 07/45] Remove more LoC references --- pom.xml | 1 + src/edu/ucsb/nceas/metacat/dataone/MNodeService.java | 3 --- .../nceas/metacat/dataone/D1NodeVersionCheckerTest.java | 6 +----- .../ucsb/nceas/metacat/dataone/MNodeReplicationTest.java | 4 ---- test/edu/ucsb/nceas/metacat/dataone/MNodeServiceTest.java | 4 ---- 5 files changed, 2 insertions(+), 16 deletions(-) diff --git a/pom.xml b/pom.xml index 97d1ade10..5c50068ac 100644 --- a/pom.xml +++ b/pom.xml @@ -443,6 +443,7 @@ org.dataone speedbagit 1.0-SNAPSHOT + jar diff --git a/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java b/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java index 42df09ba0..b64079124 100644 --- a/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java +++ b/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java @@ -175,9 +175,6 @@ import edu.ucsb.nceas.utilities.PropertyNotFoundException; import edu.ucsb.nceas.utilities.XMLUtilities; import edu.ucsb.nceas.utilities.export.HtmlToPdf; -import gov.loc.repository.bagit.Bag; -import gov.loc.repository.bagit.BagFactory; -import gov.loc.repository.bagit.writer.impl.ZipWriter; /** * Represents Metacat's implementation of the DataONE Member Node diff --git a/test/edu/ucsb/nceas/metacat/dataone/D1NodeVersionCheckerTest.java b/test/edu/ucsb/nceas/metacat/dataone/D1NodeVersionCheckerTest.java index 083eea8e3..bc52b3819 100644 --- a/test/edu/ucsb/nceas/metacat/dataone/D1NodeVersionCheckerTest.java +++ b/test/edu/ucsb/nceas/metacat/dataone/D1NodeVersionCheckerTest.java @@ -33,11 +33,7 @@ import edu.ucsb.nceas.metacat.dataone.MNodeService; import edu.ucsb.nceas.metacat.properties.SkinPropertyService; import edu.ucsb.nceas.metacat.service.ServiceService; -import edu.ucsb.nceas.utilities.IOUtil; -import gov.loc.repository.bagit.Bag; -import gov.loc.repository.bagit.BagFactory; -import gov.loc.repository.bagit.BagFile; -import gov.loc.repository.bagit.Manifest; +import edu.ucsb.nceas.utilities.IOUtil;s import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; diff --git a/test/edu/ucsb/nceas/metacat/dataone/MNodeReplicationTest.java b/test/edu/ucsb/nceas/metacat/dataone/MNodeReplicationTest.java index 205f14905..362efc0e0 100644 --- a/test/edu/ucsb/nceas/metacat/dataone/MNodeReplicationTest.java +++ b/test/edu/ucsb/nceas/metacat/dataone/MNodeReplicationTest.java @@ -35,10 +35,6 @@ import edu.ucsb.nceas.metacat.properties.SkinPropertyService; import edu.ucsb.nceas.metacat.service.ServiceService; import edu.ucsb.nceas.utilities.IOUtil; -import gov.loc.repository.bagit.Bag; -import gov.loc.repository.bagit.BagFactory; -import gov.loc.repository.bagit.BagFile; -import gov.loc.repository.bagit.Manifest; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; diff --git a/test/edu/ucsb/nceas/metacat/dataone/MNodeServiceTest.java b/test/edu/ucsb/nceas/metacat/dataone/MNodeServiceTest.java index 0ac526f76..1dc545260 100644 --- a/test/edu/ucsb/nceas/metacat/dataone/MNodeServiceTest.java +++ b/test/edu/ucsb/nceas/metacat/dataone/MNodeServiceTest.java @@ -36,10 +36,6 @@ import edu.ucsb.nceas.metacat.service.ServiceService; import edu.ucsb.nceas.metacat.util.AuthUtil; import edu.ucsb.nceas.utilities.IOUtil; -import gov.loc.repository.bagit.Bag; -import gov.loc.repository.bagit.BagFactory; -import gov.loc.repository.bagit.BagFile; -import gov.loc.repository.bagit.Manifest; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; From bc7122d42825241601c64d3beeeebef7d6df7956 Mon Sep 17 00:00:00 2001 From: Thomas Thelen Date: Wed, 24 Feb 2021 12:10:08 -0800 Subject: [PATCH 08/45] Replace InputStream references with SpeedBagIt and use ZipOutputStream for client request --- .../nceas/metacat/dataone/MNodeService.java | 12 +++--- .../metacat/dataone/v1/MNodeService.java | 3 +- .../restservice/v1/MNResourceHandler.java | 8 ++-- .../restservice/v2/MNResourceHandler.java | 37 +++++++++++++------ .../metacat/dataone/MNodeServiceTest.java | 3 +- 5 files changed, 41 insertions(+), 22 deletions(-) diff --git a/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java b/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java index b64079124..be0966aa9 100644 --- a/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java +++ b/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java @@ -38,6 +38,8 @@ import java.math.BigInteger; import java.net.URISyntaxException; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.nio.file.Paths; import java.security.NoSuchAlgorithmException; import java.sql.SQLException; @@ -2551,7 +2553,7 @@ private List lookupOreFor(Session session, Identifier guid) { * @throws NotFound */ @Override - public InputStream getPackage(Session session, ObjectFormatIdentifier formatId, + public SpeedBagIt getPackage(Session session, ObjectFormatIdentifier formatId, Identifier pid) throws InvalidToken, ServiceFailure, NotAuthorized, InvalidRequest, NotImplemented, NotFound { if(formatId == null) { @@ -2566,7 +2568,7 @@ public InputStream getPackage(Session session, ObjectFormatIdentifier formatId, pid = sid; } // Create a bag that is version 0.97 and has tag files that contain MD5 checksums - SpeedBagIt speedBag = new SpeedBagIt("0.97", "MD5"); + SpeedBagIt speedBag = new SpeedBagIt(0.97, "MD5"); // The pids of all of the objects in the package List packagePids = new ArrayList(); @@ -2652,7 +2654,7 @@ public InputStream getPackage(Session session, ObjectFormatIdentifier formatId, // Create the pdf File that HtmlToPdf will write to String pdfFileName = metadataID.getValue().replaceAll("[^a-zA-Z0-9\\-\\.]", "_") + "-METADATA.pdf"; File pdfFile = new File(tmpDir, pdfFileName); - File pdfFile = File.createTempFile("metadata", ".pdf", tmpDir); + pdfFile = File.createTempFile("metadata", ".pdf", tmpDir); // write the HTML file File htmlFile = File.createTempFile("metadata", ".html", tmpDir); @@ -2676,7 +2678,7 @@ public InputStream getPackage(Session session, ObjectFormatIdentifier formatId, tmpDir.delete(); // Add the pdf to the bag - speedBag.addFile(pdfInputStream, Paths.get("data", pdfFileName)); + speedBag.addFile(pdfInputStream, Paths.get("data", pdfFileName), false); // Create a record in the pid mapping file pidMapping.append(metadataID.getValue() + " (pdf)" + "\t" + "data/" + pdfFile.getName() + "\n"); @@ -2779,7 +2781,7 @@ public InputStream getPackage(Session session, ObjectFormatIdentifier formatId, // Add the stream of the file to the bag object & write to the pid mapping file InputStream entryInputStream = this.get(session, entryPid); speedBag.addFile(entryInputStream, Paths.get("data", filename)); - pidMapping.append(entryPid.getValue() + "\t" + "data/" + tempFile.getName() + "\n"); + pidMapping.append(entryPid.getValue() + "\t" + "data/" + filename + "\n"); } // Get a stream to the pid mapping file and add it as a tag file, in the bag root diff --git a/src/edu/ucsb/nceas/metacat/dataone/v1/MNodeService.java b/src/edu/ucsb/nceas/metacat/dataone/v1/MNodeService.java index 4d655459d..03bce3300 100644 --- a/src/edu/ucsb/nceas/metacat/dataone/v1/MNodeService.java +++ b/src/edu/ucsb/nceas/metacat/dataone/v1/MNodeService.java @@ -64,6 +64,7 @@ import org.dataone.service.types.v1_1.QueryEngineDescription; import org.dataone.service.types.v1_1.QueryEngineList; import org.dataone.service.types.v2.TypeFactory; +import org.dataone.speedbagit.SpeedBagIt; import edu.ucsb.nceas.metacat.IdentifierManager; import edu.ucsb.nceas.metacat.dataone.convert.LogV2toV1Converter; @@ -572,7 +573,7 @@ public InputStream view(Session session, String format, Identifier pid) return impl.view(session, format, pid); } - public InputStream getPackage(Session session, ObjectFormatIdentifier formatId, + public SpeedBagIt getPackage(Session session, ObjectFormatIdentifier formatId, Identifier pid) throws InvalidToken, ServiceFailure, NotAuthorized, InvalidRequest, NotImplemented, NotFound { String serviceFailure = "2871"; diff --git a/src/edu/ucsb/nceas/metacat/restservice/v1/MNResourceHandler.java b/src/edu/ucsb/nceas/metacat/restservice/v1/MNResourceHandler.java index 1b569474d..04f7ba5b6 100644 --- a/src/edu/ucsb/nceas/metacat/restservice/v1/MNResourceHandler.java +++ b/src/edu/ucsb/nceas/metacat/restservice/v1/MNResourceHandler.java @@ -33,6 +33,7 @@ import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.zip.ZipOutputStream; import javax.servlet.ServletContext; import javax.servlet.http.HttpServletRequest; @@ -77,6 +78,7 @@ import org.dataone.service.util.DateTimeMarshaller; import org.dataone.service.util.ExceptionHandler; import org.dataone.service.util.TypeMarshaller; +import org.dataone.speedbagit.SpeedBagIt; import org.xml.sax.SAXException; import edu.ucsb.nceas.metacat.common.query.stream.ContentTypeInputStream; @@ -1234,7 +1236,7 @@ protected void getPackage(String pid) throws InvalidToken, ServiceFailure, NotAu Identifier id = new Identifier(); id.setValue(pid); - InputStream is = MNodeService.getInstance(request).getPackage(session, null, id); + SpeedBagIt speedBag = MNodeService.getInstance(request).getPackage(session, null, id); //Use the pid as the file name prefix, replacing all non-word characters String filename = pid.getValue().replaceAll("\\W", "_"); @@ -1242,10 +1244,10 @@ protected void getPackage(String pid) throws InvalidToken, ServiceFailure, NotAu response.setHeader("Content-Disposition", "inline; filename=\"" + filename+"\""); response.setContentType("application/zip"); response.setStatus(200); - OutputStream out = response.getOutputStream(); + ZipOutputStream out = response.getOutputStream(); // write it to the output stream - IOUtils.copyLarge(is, out); + speedBag.stream(out); } protected void publish(String pid) throws InvalidToken, ServiceFailure, diff --git a/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java b/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java index 90b78e0ca..950c3aadf 100644 --- a/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java +++ b/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java @@ -36,6 +36,7 @@ import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.zip.ZipOutputStream; import javax.servlet.ServletContext; import javax.servlet.http.HttpServletRequest; @@ -86,6 +87,7 @@ import org.dataone.service.util.DateTimeMarshaller; import org.dataone.service.util.ExceptionHandler; import org.dataone.service.util.TypeMarshaller; +import org.dataone.speedbagit.SpeedBagIt; import org.xml.sax.SAXException; import edu.ucsb.nceas.metacat.MetaCatServlet; @@ -1514,21 +1516,20 @@ else if(name.equals("count") && value != null) * @throws InvalidRequest */ protected void getPackage(String format, String pid) throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented, IOException, InvalidRequest { - long start = System.currentTimeMillis(); - Identifier id = new Identifier(); - id.setValue(pid); - ObjectFormatIdentifier formatId = null; - if (format != null) { - formatId = new ObjectFormatIdentifier(); - formatId.setValue(format); - } - InputStream is = null; - try { - is = MNodeService.getInstance(request).getPackage(session, formatId, id); + + long start = System.currentTimeMillis(); + Identifier id = new Identifier(); + id.setValue(pid); + ObjectFormatIdentifier formatId = null; + if (format != null) { + formatId = new ObjectFormatIdentifier(); + formatId.setValue(format); + } + try{ + SpeedBagIt speedBag = MNodeService.getInstance(request).getPackage(session, formatId , id); //Use the pid as the file name prefix, replacing all non-word characters String filename = pid.replaceAll("\\W", "_") + ".zip"; - response.setHeader("Content-Disposition", "inline; filename=\"" + filename + "\""); response.setContentType("application/zip"); response.setStatus(200); @@ -1543,6 +1544,18 @@ protected void getPackage(String format, String pid) throws InvalidToken, Servic IOUtils.closeQuietly(is); } } +======= + response.setHeader("Content-Disposition", "inline; filename=\"" + filename+"\""); + response.setContentType("application/zip"); + response.setStatus(200); + ZipOutputStream out = response.getOutputStream(); + + // write it to the output stream + speedBag.stream(out); + long end = System.currentTimeMillis(); + logMetacat.info(Settings.PERFORMANCELOG + pid + Settings.PERFORMANCELOG_GET_PACKAGE_METHOD + " Total getPackage method" + Settings.PERFORMANCELOG_DURATION + (end-start)/1000); + } +>>>>>>> Replace InputStream references with SpeedBagIt and use ZipOutputStream for client request protected void publish(String pid) throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented, IOException, diff --git a/test/edu/ucsb/nceas/metacat/dataone/MNodeServiceTest.java b/test/edu/ucsb/nceas/metacat/dataone/MNodeServiceTest.java index 1dc545260..b8d515f18 100644 --- a/test/edu/ucsb/nceas/metacat/dataone/MNodeServiceTest.java +++ b/test/edu/ucsb/nceas/metacat/dataone/MNodeServiceTest.java @@ -100,6 +100,7 @@ import org.dataone.service.types.v1.SubjectInfo; import org.dataone.service.types.v1.util.ChecksumUtil; import org.dataone.service.types.v2.SystemMetadata; +import org.dataone.speedbagit.SpeedBagIt; import org.dspace.foresite.ResourceMap; import org.junit.After; import org.junit.Before; @@ -1756,7 +1757,7 @@ public void testGetPackage() { Identifier pid = MNodeService.getInstance(request).create(session, guid, object, sysmeta); ObjectFormatIdentifier format = new ObjectFormatIdentifier(); format.setValue("application/bagit-097"); - InputStream bagStream = MNodeService.getInstance(request).getPackage(session, format, pid); + SpeedBagIt bagStream = MNodeService.getInstance(request).getPackage(session, format, pid); /*File bagFile = File.createTempFile("bagit.", ".zip"); IOUtils.copy(bagStream, new FileOutputStream(bagFile)); BagFactory bagFactory = new BagFactory(); From a94f739d31663d0574312af3d19b3496e2f6009b Mon Sep 17 00:00:00 2001 From: Thomas Thelen Date: Wed, 24 Feb 2021 12:58:00 -0800 Subject: [PATCH 09/45] Fix API calls to SpeedBagIt and misc other Java classes --- .../nceas/metacat/dataone/MNodeService.java | 10 +- .../restservice/v1/MNResourceHandler.java | 4 +- .../restservice/v2/MNResourceHandler.java | 2476 ++++++++--------- 3 files changed, 1229 insertions(+), 1261 deletions(-) diff --git a/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java b/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java index be0966aa9..0dad86a15 100644 --- a/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java +++ b/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java @@ -2670,7 +2670,7 @@ public SpeedBagIt getPackage(Session session, ObjectFormatIdentifier formatId, cssDir.delete(); // Load the PDF file into memory so that it can be deleted from disk - byte[] pdfMetadata = Files.readAllBytes(pdfFile.getAbsolutePath()); + byte[] pdfMetadata = Files.readAllBytes(Paths.get(pdfFile.getAbsolutePath())); InputStream pdfInputStream = new ByteArrayInputStream(pdfMetadata); // Now that the PDF file has been loaded into memory, delete it from disk @@ -2678,7 +2678,7 @@ public SpeedBagIt getPackage(Session session, ObjectFormatIdentifier formatId, tmpDir.delete(); // Add the pdf to the bag - speedBag.addFile(pdfInputStream, Paths.get("data", pdfFileName), false); + speedBag.addFile(pdfInputStream, Paths.get("data/"+ pdfFileName).toString(), false); // Create a record in the pid mapping file pidMapping.append(metadataID.getValue() + " (pdf)" + "\t" + "data/" + pdfFile.getName() + "\n"); @@ -2780,12 +2780,12 @@ public SpeedBagIt getPackage(Session session, ObjectFormatIdentifier formatId, // Add the stream of the file to the bag object & write to the pid mapping file InputStream entryInputStream = this.get(session, entryPid); - speedBag.addFile(entryInputStream, Paths.get("data", filename)); - pidMapping.append(entryPid.getValue() + "\t" + "data/" + filename + "\n"); + speedBag.addFile(entryInputStream, Paths.get("data/", fileName).toString(), false); + pidMapping.append(entryPid.getValue() + "\t" + "data" + fileName + "\n"); } // Get a stream to the pid mapping file and add it as a tag file, in the bag root - ByteArrayInputStream pidFile = ByteArrayInputStream(pidMapping.toString().getBytes(StandardCharsets.UTF_8)); + ByteArrayInputStream pidFile = new ByteArrayInputStream(pidMapping.toString().getBytes(StandardCharsets.UTF_8)); speedBag.addFile(pidFile, "pid-mapping.txt", true); } catch (IOException e) { // report as service failure diff --git a/src/edu/ucsb/nceas/metacat/restservice/v1/MNResourceHandler.java b/src/edu/ucsb/nceas/metacat/restservice/v1/MNResourceHandler.java index 04f7ba5b6..2f1fad6f4 100644 --- a/src/edu/ucsb/nceas/metacat/restservice/v1/MNResourceHandler.java +++ b/src/edu/ucsb/nceas/metacat/restservice/v1/MNResourceHandler.java @@ -1239,12 +1239,12 @@ protected void getPackage(String pid) throws InvalidToken, ServiceFailure, NotAu SpeedBagIt speedBag = MNodeService.getInstance(request).getPackage(session, null, id); //Use the pid as the file name prefix, replacing all non-word characters - String filename = pid.getValue().replaceAll("\\W", "_"); + String filename = pid.replaceAll("\\W", "_"); response.setHeader("Content-Disposition", "inline; filename=\"" + filename+"\""); response.setContentType("application/zip"); response.setStatus(200); - ZipOutputStream out = response.getOutputStream(); + ZipOutputStream out = new ZipOutputStream(response.getOutputStream()); // write it to the output stream speedBag.stream(out); diff --git a/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java b/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java index 950c3aadf..ed026db66 100644 --- a/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java +++ b/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java @@ -147,763 +147,762 @@ */ public class MNResourceHandler extends D1ResourceHandler { - // MN-specific API Resources - protected static final String RESOURCE_MONITOR = "monitor"; - protected static final String RESOURCE_REPLICATE = "replicate"; - protected static final String RESOURCE_REPLICAS = "replica"; - protected static final String RESOURCE_NODE = "node"; - protected static final String RESOURCE_ERROR = "error"; - protected static final String RESOURCE_META_CHANGED = "dirtySystemMetadata"; - protected static final String RESOURCE_GENERATE_ID = "generate"; - protected static final String RESOURCE_PUBLISH = "publish"; - protected static final String RESOURCE_PACKAGE = "packages"; - protected static final String RESOURCE_TOKEN = "token"; - protected static final String RESOURCE_WHOAMI = "whoami"; + // MN-specific API Resources + protected static final String RESOURCE_MONITOR = "monitor"; + protected static final String RESOURCE_REPLICATE = "replicate"; + protected static final String RESOURCE_REPLICAS = "replica"; + protected static final String RESOURCE_NODE = "node"; + protected static final String RESOURCE_ERROR = "error"; + protected static final String RESOURCE_META_CHANGED = "dirtySystemMetadata"; + protected static final String RESOURCE_GENERATE_ID = "generate"; + protected static final String RESOURCE_PUBLISH = "publish"; + protected static final String RESOURCE_PACKAGE = "packages"; + protected static final String RESOURCE_TOKEN = "token"; + protected static final String RESOURCE_WHOAMI = "whoami"; - - - // shared executor + // shared executor private static ExecutorService executor = null; static { // use a shared executor service with nThreads == one less than available processors - int availableProcessors = Runtime.getRuntime().availableProcessors(); - int nThreads = availableProcessors * 1; - nThreads--; - nThreads = Math.max(1, nThreads); - executor = Executors.newFixedThreadPool(nThreads); + int availableProcessors = Runtime.getRuntime().availableProcessors(); + int nThreads = availableProcessors * 1; + nThreads--; + nThreads = Math.max(1, nThreads); + executor = Executors.newFixedThreadPool(nThreads); } - - /** - * Initializes new instance by setting servlet context,request and response - * */ - public MNResourceHandler(ServletContext servletContext, - HttpServletRequest request, HttpServletResponse response) { - super(servletContext, request, response); - logMetacat = LogFactory.getLog(MNResourceHandler.class); - } - - @Override - protected boolean isD1Enabled() { - - boolean enabled = false; - try { + + /** + * Initializes new instance by setting servlet context,request and response + */ + public MNResourceHandler(ServletContext servletContext, + HttpServletRequest request, HttpServletResponse response) { + super(servletContext, request, response); + logMetacat = LogFactory.getLog(MNResourceHandler.class); + } + + @Override + protected boolean isD1Enabled() { + + boolean enabled = false; + try { enabled = Boolean.parseBoolean(PropertyService.getProperty("dataone.mn.services.enabled")); } catch (PropertyNotFoundException e) { logMetacat.error("Could not check if DataONE is enabled: " + e.getMessage()); } - - return enabled; - } - /** - * This function is called from REST API servlet and handles each request to the servlet - * - * @param httpVerb (GET, POST, PUT or DELETE) - */ - @Override - public void handle(byte httpVerb) { - // prepare the handler - super.handle(httpVerb); - - try { - - // only service requests if we have D1 configured - if (!isD1Enabled()) { - ServiceFailure se = new ServiceFailure("0000", "DataONE services are not enabled on this node"); - serializeException(se, response.getOutputStream()); - return; - } - - // get the resource - String resource = request.getPathInfo(); - resource = resource.substring(resource.indexOf("/") + 1); - - // default to node info - if (resource.equals("")) { - resource = RESOURCE_NODE; - } - - // get the rest of the path info - String extra = null; - - logMetacat.info("MNResourceHandler.handle - V2 handling verb " + httpVerb + " request with resource '" + resource + "'"); - logMetacat.debug("resource: '" + resource + "'"); - boolean status = false; - - if (resource != null) { - - if (resource.startsWith(RESOURCE_NODE)) { - // node response - node(); - status = true; - } else if (resource.startsWith(RESOURCE_TOKEN)) { - logMetacat.debug("Using resource 'token'"); - // get - if (httpVerb == GET) { - // after the command - getToken(); - status = true; - } - - } else if (resource.startsWith(RESOURCE_WHOAMI)) { - logMetacat.debug("Using resource 'whoami'"); - // get - if (httpVerb == GET) { - // after the command - whoami(); - status = true; - } - - } else if (resource.startsWith(RESOURCE_IS_AUTHORIZED)) { - if (httpVerb == GET) { - // after the command - extra = parseTrailing(resource, RESOURCE_IS_AUTHORIZED); - extra = decode(extra); - // check the access rules - isAuthorized(extra); - status = true; - logMetacat.debug("done getting access"); - } - } else if (resource.startsWith(RESOURCE_META)) { - logMetacat.debug("Using resource 'meta'"); - // get - if (httpVerb == GET) { - logMetacat.debug("Using resource 'meta' for GET"); - // after the command - extra = parseTrailing(resource, RESOURCE_META); - extra = decode(extra); - getSystemMetadataObject(extra); - status = true; - } else if (httpVerb == PUT) { - logMetacat.debug("Using resource 'meta' for PUT"); - updateSystemMetadata(); - status = true; - } - - } else if (resource.startsWith(RESOURCE_OBJECTS)) { - logMetacat.debug("Using resource 'object'"); - // after the command - extra = parseTrailing(resource, RESOURCE_OBJECTS); - extra = decode(extra); - logMetacat.debug("objectId: " + extra); - logMetacat.debug("verb:" + httpVerb); - - if (httpVerb == GET) { - getObject(extra); - status = true; - } else if (httpVerb == POST) { - // part of the params, not the URL - putObject(null, FUNCTION_NAME_INSERT); - status = true; - } else if (httpVerb == PUT) { - putObject(extra, FUNCTION_NAME_UPDATE); - status = true; - } else if (httpVerb == DELETE) { - deleteObject(extra); - status = true; - } else if (httpVerb == HEAD) { - describeObject(extra); - status = true; - } - - } else if (resource.startsWith(RESOURCE_LOG)) { - logMetacat.debug("Using resource 'log'"); - // handle log events - if (httpVerb == GET) { - getLog(); - status = true; - } - } else if (resource.startsWith(Constants.RESOURCE_ARCHIVE)) { - logMetacat.debug("Using resource " + Constants.RESOURCE_ARCHIVE); - // handle archive events - if (httpVerb == PUT) { - extra = parseTrailing(resource, Constants.RESOURCE_ARCHIVE); - extra = decode(extra); - archive(extra); - status = true; - } - } else if (resource.startsWith(Constants.RESOURCE_CHECKSUM)) { - logMetacat.debug("Using resource 'checksum'"); - // handle checksum requests - if (httpVerb == GET) { - // after the command - extra = parseTrailing(resource, Constants.RESOURCE_CHECKSUM); - extra = decode(extra); - checksum(extra); - status = true; - } - } else if (resource.startsWith(RESOURCE_MONITOR)) { - // there are various parts to monitoring - if (httpVerb == GET) { - // after the command - extra = parseTrailing(resource, RESOURCE_MONITOR); - extra = decode(extra); - // ping - if (extra.toLowerCase().equals("ping")) { - logMetacat.debug("processing ping request"); - Date result = MNodeService.getInstance(request).ping(); - // TODO: send to output - status = true; - - } else if (extra.toLowerCase().equals("status")) { - logMetacat.debug("processing status request"); - getStatus(); - status = true; - } - - } - } else if (resource.startsWith(RESOURCE_REPLICATE)) { - if (httpVerb == POST) { - logMetacat.debug("processing replicate request"); - replicate(); - status = true; - } - } else if (resource.startsWith(RESOURCE_ERROR)) { - // sync error - if (httpVerb == POST) { - syncError(); - status = true; - } - } else if (resource.startsWith(RESOURCE_META_CHANGED)) { - // system metadata changed - if (httpVerb == POST) { - systemMetadataChanged(); - status = true; - } - } else if (resource.startsWith(RESOURCE_REPLICAS)) { - // get replica - if (httpVerb == GET) { - extra = parseTrailing(resource, RESOURCE_REPLICAS); - extra = decode(extra); - getReplica(extra); - status = true; - } - } else if (resource.startsWith(RESOURCE_QUERY)) { - logMetacat.debug("Using resource " + RESOURCE_QUERY); - // after the command - extra = parseTrailing(resource, RESOURCE_QUERY); - logMetacat.debug("query extra: " + extra); - - String engine = null; - String query = null; - - if (extra != null) { - // get the engine - int engineIndex = extra.length(); - if (extra.indexOf("/") > -1) { - engineIndex = extra.indexOf("/"); - } - engine = extra.substring(0, engineIndex); - engine = decode(engine); - logMetacat.debug("query engine: " + engine); - - // check the query string first - query = request.getQueryString(); - - // if null, look at the whole endpoint - if (query == null) { - // get the query if it is there - query = extra.substring(engineIndex, extra.length()); - if (query != null && query.length() == 0) { - query = null; - } else { - if (query.startsWith("/")) { - query = query.substring(1); - } - } - } - query = decode(query); - logMetacat.debug("query: " + query); - - } - logMetacat.debug("verb:" + httpVerb); - if (httpVerb == GET) { - doQuery(engine, query); - status = true; - } else if (httpVerb == POST) { - doPostQuery(engine); - status = true; - } - } else if (resource.startsWith(RESOURCE_GENERATE_ID)) { - // generate an id - if (httpVerb == POST) { - generateIdentifier(); - status = true; - } - } else if (resource.startsWith(RESOURCE_PUBLISH)) { - logMetacat.debug("Using resource: " + RESOURCE_PUBLISH); - // PUT - if (httpVerb == PUT) { - // after the command - extra = parseTrailing(resource, RESOURCE_PUBLISH); - extra = decode(extra); - publish(extra); - status = true; - } - } else if (resource.startsWith(RESOURCE_PACKAGE)) { - logMetacat.debug("Using resource: " + RESOURCE_PACKAGE); - // after the command - extra = parseTrailing(resource, RESOURCE_PACKAGE); - - String format = null; - String pid = null; - - if (extra != null) { - // get the format - int formatIndex = extra.length(); - if (extra.indexOf("/") > -1) { - formatIndex = extra.indexOf("/"); - } - format = extra.substring(0, formatIndex); - format = decode(format); - logMetacat.debug("package format: " + format); - - // get the pid if it is there - pid = extra.substring(formatIndex, extra.length()); - if (pid != null && pid.length() == 0) { - pid = null; - } else { - if (pid.startsWith("/")) { - pid = pid.substring(1); - } - } - pid = decode(pid); - logMetacat.debug("pid: " + pid); - - } - - // get - if (httpVerb == GET) { - - getPackage(format, pid); - status = true; - } - } else if (resource.startsWith(RESOURCE_VIEWS)) { - logMetacat.debug("Using resource " + RESOURCE_VIEWS); - // after the command - extra = parseTrailing(resource, RESOURCE_VIEWS); - logMetacat.debug("view extra: " + extra); - - String format = null; - String pid = null; - - if (extra != null) { - // get the format - int formatIndex = extra.length(); - if (extra.indexOf("/") > -1) { - formatIndex = extra.indexOf("/"); - } - format = extra.substring(0, formatIndex); - format = decode(format); - logMetacat.debug("view format: " + format); - - // get the pid if it is there - pid = extra.substring(formatIndex, extra.length()); - if (pid != null && pid.length() == 0) { - pid = null; - } else { - if (pid.startsWith("/")) { - pid = pid.substring(1); - } - } - pid = decode(pid); - logMetacat.debug("pid: " + pid); - - } - logMetacat.debug("verb:" + httpVerb); - if (httpVerb == GET) { - doViews(format, pid); - status = true; - } - } else { - throw new InvalidRequest("0000", "No resource matched for " + resource); - } - - if (!status) { - throw new ServiceFailure("0000", "Unknown error, status = " + status); - } - } else { - throw new InvalidRequest("0000", "No resource matched for " + resource); - } - } catch (BaseException be) { - // report Exceptions as clearly as possible - OutputStream out = null; + return enabled; + } + + /** + * This function is called from REST API servlet and handles each request to the servlet + * + * @param httpVerb (GET, POST, PUT or DELETE) + */ + @Override + public void handle(byte httpVerb) { + // prepare the handler + super.handle(httpVerb); + + try { + + // only service requests if we have D1 configured + if (!isD1Enabled()) { + ServiceFailure se = new ServiceFailure("0000", "DataONE services are not enabled on this node"); + serializeException(se, response.getOutputStream()); + return; + } + + // get the resource + String resource = request.getPathInfo(); + resource = resource.substring(resource.indexOf("/") + 1); + + // default to node info + if (resource.equals("")) { + resource = RESOURCE_NODE; + } + + // get the rest of the path info + String extra = null; + + logMetacat.info("MNResourceHandler.handle - V2 handling verb " + httpVerb + " request with resource '" + resource + "'"); + logMetacat.debug("resource: '" + resource + "'"); + boolean status = false; + + if (resource != null) { + + if (resource.startsWith(RESOURCE_NODE)) { + // node response + node(); + status = true; + } else if (resource.startsWith(RESOURCE_TOKEN)) { + logMetacat.debug("Using resource 'token'"); + // get + if (httpVerb == GET) { + // after the command + getToken(); + status = true; + } + + } else if (resource.startsWith(RESOURCE_WHOAMI)) { + logMetacat.debug("Using resource 'whoami'"); + // get + if (httpVerb == GET) { + // after the command + whoami(); + status = true; + } + + } else if (resource.startsWith(RESOURCE_IS_AUTHORIZED)) { + if (httpVerb == GET) { + // after the command + extra = parseTrailing(resource, RESOURCE_IS_AUTHORIZED); + extra = decode(extra); + // check the access rules + isAuthorized(extra); + status = true; + logMetacat.debug("done getting access"); + } + } else if (resource.startsWith(RESOURCE_META)) { + logMetacat.debug("Using resource 'meta'"); + // get + if (httpVerb == GET) { + logMetacat.debug("Using resource 'meta' for GET"); + // after the command + extra = parseTrailing(resource, RESOURCE_META); + extra = decode(extra); + getSystemMetadataObject(extra); + status = true; + } else if (httpVerb == PUT) { + logMetacat.debug("Using resource 'meta' for PUT"); + updateSystemMetadata(); + status = true; + } + + } else if (resource.startsWith(RESOURCE_OBJECTS)) { + logMetacat.debug("Using resource 'object'"); + // after the command + extra = parseTrailing(resource, RESOURCE_OBJECTS); + extra = decode(extra); + logMetacat.debug("objectId: " + extra); + logMetacat.debug("verb:" + httpVerb); + + if (httpVerb == GET) { + getObject(extra); + status = true; + } else if (httpVerb == POST) { + // part of the params, not the URL + putObject(null, FUNCTION_NAME_INSERT); + status = true; + } else if (httpVerb == PUT) { + putObject(extra, FUNCTION_NAME_UPDATE); + status = true; + } else if (httpVerb == DELETE) { + deleteObject(extra); + status = true; + } else if (httpVerb == HEAD) { + describeObject(extra); + status = true; + } + + } else if (resource.startsWith(RESOURCE_LOG)) { + logMetacat.debug("Using resource 'log'"); + // handle log events + if (httpVerb == GET) { + getLog(); + status = true; + } + } else if (resource.startsWith(Constants.RESOURCE_ARCHIVE)) { + logMetacat.debug("Using resource " + Constants.RESOURCE_ARCHIVE); + // handle archive events + if (httpVerb == PUT) { + extra = parseTrailing(resource, Constants.RESOURCE_ARCHIVE); + extra = decode(extra); + archive(extra); + status = true; + } + } else if (resource.startsWith(Constants.RESOURCE_CHECKSUM)) { + logMetacat.debug("Using resource 'checksum'"); + // handle checksum requests + if (httpVerb == GET) { + // after the command + extra = parseTrailing(resource, Constants.RESOURCE_CHECKSUM); + extra = decode(extra); + checksum(extra); + status = true; + } + } else if (resource.startsWith(RESOURCE_MONITOR)) { + // there are various parts to monitoring + if (httpVerb == GET) { + // after the command + extra = parseTrailing(resource, RESOURCE_MONITOR); + extra = decode(extra); + // ping + if (extra.toLowerCase().equals("ping")) { + logMetacat.debug("processing ping request"); + Date result = MNodeService.getInstance(request).ping(); + // TODO: send to output + status = true; + + } else if (extra.toLowerCase().equals("status")) { + logMetacat.debug("processing status request"); + getStatus(); + status = true; + } + + } + } else if (resource.startsWith(RESOURCE_REPLICATE)) { + if (httpVerb == POST) { + logMetacat.debug("processing replicate request"); + replicate(); + status = true; + } + } else if (resource.startsWith(RESOURCE_ERROR)) { + // sync error + if (httpVerb == POST) { + syncError(); + status = true; + } + } else if (resource.startsWith(RESOURCE_META_CHANGED)) { + // system metadata changed + if (httpVerb == POST) { + systemMetadataChanged(); + status = true; + } + } else if (resource.startsWith(RESOURCE_REPLICAS)) { + // get replica + if (httpVerb == GET) { + extra = parseTrailing(resource, RESOURCE_REPLICAS); + extra = decode(extra); + getReplica(extra); + status = true; + } + } else if (resource.startsWith(RESOURCE_QUERY)) { + logMetacat.debug("Using resource " + RESOURCE_QUERY); + // after the command + extra = parseTrailing(resource, RESOURCE_QUERY); + logMetacat.debug("query extra: " + extra); + + String engine = null; + String query = null; + + if (extra != null) { + // get the engine + int engineIndex = extra.length(); + if (extra.indexOf("/") > -1) { + engineIndex = extra.indexOf("/"); + } + engine = extra.substring(0, engineIndex); + engine = decode(engine); + logMetacat.debug("query engine: " + engine); + + // check the query string first + query = request.getQueryString(); + + // if null, look at the whole endpoint + if (query == null) { + // get the query if it is there + query = extra.substring(engineIndex, extra.length()); + if (query != null && query.length() == 0) { + query = null; + } else { + if (query.startsWith("/")) { + query = query.substring(1); + } + } + } + query = decode(query); + logMetacat.debug("query: " + query); + + } + logMetacat.debug("verb:" + httpVerb); + if (httpVerb == GET) { + doQuery(engine, query); + status = true; + } else if (httpVerb == POST) { + doPostQuery(engine); + status = true; + } + } else if (resource.startsWith(RESOURCE_GENERATE_ID)) { + // generate an id + if (httpVerb == POST) { + generateIdentifier(); + status = true; + } + } else if (resource.startsWith(RESOURCE_PUBLISH)) { + logMetacat.debug("Using resource: " + RESOURCE_PUBLISH); + // PUT + if (httpVerb == PUT) { + // after the command + extra = parseTrailing(resource, RESOURCE_PUBLISH); + extra = decode(extra); + publish(extra); + status = true; + } + } else if (resource.startsWith(RESOURCE_PACKAGE)) { + logMetacat.debug("Using resource: " + RESOURCE_PACKAGE); + // after the command + extra = parseTrailing(resource, RESOURCE_PACKAGE); + + String format = null; + String pid = null; + + if (extra != null) { + // get the format + int formatIndex = extra.length(); + if (extra.indexOf("/") > -1) { + formatIndex = extra.indexOf("/"); + } + format = extra.substring(0, formatIndex); + format = decode(format); + logMetacat.debug("package format: " + format); + + // get the pid if it is there + pid = extra.substring(formatIndex, extra.length()); + if (pid != null && pid.length() == 0) { + pid = null; + } else { + if (pid.startsWith("/")) { + pid = pid.substring(1); + } + } + pid = decode(pid); + logMetacat.debug("pid: " + pid); + + } + + // get + if (httpVerb == GET) { + + getPackage(format, pid); + status = true; + } + } else if (resource.startsWith(RESOURCE_VIEWS)) { + logMetacat.debug("Using resource " + RESOURCE_VIEWS); + // after the command + extra = parseTrailing(resource, RESOURCE_VIEWS); + logMetacat.debug("view extra: " + extra); + + String format = null; + String pid = null; + + if (extra != null) { + // get the format + int formatIndex = extra.length(); + if (extra.indexOf("/") > -1) { + formatIndex = extra.indexOf("/"); + } + format = extra.substring(0, formatIndex); + format = decode(format); + logMetacat.debug("view format: " + format); + + // get the pid if it is there + pid = extra.substring(formatIndex, extra.length()); + if (pid != null && pid.length() == 0) { + pid = null; + } else { + if (pid.startsWith("/")) { + pid = pid.substring(1); + } + } + pid = decode(pid); + logMetacat.debug("pid: " + pid); + + } + logMetacat.debug("verb:" + httpVerb); + if (httpVerb == GET) { + doViews(format, pid); + status = true; + } + } else { + throw new InvalidRequest("0000", "No resource matched for " + resource); + } + + if (!status) { + throw new ServiceFailure("0000", "Unknown error, status = " + status); + } + } else { + throw new InvalidRequest("0000", "No resource matched for " + resource); + } + } catch (BaseException be) { + // report Exceptions as clearly as possible + OutputStream out = null; try { out = response.getOutputStream(); } catch (IOException e) { logMetacat.error("Could not get output stream from response", e); } - serializeException(be, out); - } catch (Exception e) { - // report Exceptions as clearly and generically as possible - logMetacat.error(e.getClass() + ": " + e.getMessage(), e); - OutputStream out = null; + serializeException(be, out); + } catch (Exception e) { + // report Exceptions as clearly and generically as possible + logMetacat.error(e.getClass() + ": " + e.getMessage(), e); + OutputStream out = null; try { out = response.getOutputStream(); } catch (IOException ioe) { logMetacat.error("Could not get output stream from response", ioe); } ServiceFailure se = new ServiceFailure("0000", e.getMessage()); - serializeException(se, out); - } - } - + serializeException(se, out); + } + } + + + private void doQuery(String engine, String query) { - private void doQuery(String engine, String query) { - OutputStream out = null; - try { - // NOTE: we set the session explicitly for the MNode instance since these methods do not provide a parameter - if (engine == null) { - // just looking for list of engines - MNodeService mnode = MNodeService.getInstance(request); - mnode.setSession(session); - QueryEngineList qel = mnode.listQueryEngines(session); - response.setContentType("text/xml"); - response.setStatus(200); - out = response.getOutputStream(); - TypeMarshaller.marshalTypeToOutputStream(qel, out); - IOUtils.closeQuietly(out); - return; - } else { - if (query != null) { - long start = System.currentTimeMillis(); - MNodeService mnode = MNodeService.getInstance(request); - mnode.setSession(session); - InputStream stream = mnode.query(session, engine, query); - - // set the content-type if we have it from the implementation - if (stream instanceof ContentTypeInputStream) { - //response.setContentType("application/octet-stream"); - //response.setContentType("text/xml"); - response.setContentType(((ContentTypeInputStream) stream).getContentType()); - } - response.setStatus(200); - out = response.getOutputStream(); - // write the results to the output stream - IOUtils.copyLarge(stream, out); - long end = System.currentTimeMillis(); - logMetacat.info(Settings.PERFORMANCELOG + Settings.PERFORMANCELOG_QUERY_METHOD + query + " Total query method" + Settings.PERFORMANCELOG_DURATION + (end-start)/1000); - IOUtils.closeQuietly(out); - return; - } else { - MNodeService mnode = MNodeService.getInstance(request); - mnode.setSession(session); - QueryEngineDescription qed = mnode.getQueryEngineDescription(session, engine); - response.setContentType("text/xml"); - response.setStatus(200); - out = response.getOutputStream(); - TypeMarshaller.marshalTypeToOutputStream(qed, out); - IOUtils.closeQuietly(out); - return; - } - } - - - } catch (BaseException be) { - // report Exceptions as clearly as possible + try { + // NOTE: we set the session explicitly for the MNode instance since these methods do not provide a parameter + if (engine == null) { + // just looking for list of engines + MNodeService mnode = MNodeService.getInstance(request); + mnode.setSession(session); + QueryEngineList qel = mnode.listQueryEngines(session); + response.setContentType("text/xml"); + response.setStatus(200); + out = response.getOutputStream(); + TypeMarshaller.marshalTypeToOutputStream(qel, out); + IOUtils.closeQuietly(out); + return; + } else { + if (query != null) { + long start = System.currentTimeMillis(); + MNodeService mnode = MNodeService.getInstance(request); + mnode.setSession(session); + InputStream stream = mnode.query(session, engine, query); + + // set the content-type if we have it from the implementation + if (stream instanceof ContentTypeInputStream) { + //response.setContentType("application/octet-stream"); + //response.setContentType("text/xml"); + response.setContentType(((ContentTypeInputStream) stream).getContentType()); + } + response.setStatus(200); + out = response.getOutputStream(); + // write the results to the output stream + IOUtils.copyLarge(stream, out); + long end = System.currentTimeMillis(); + logMetacat.info(Settings.PERFORMANCELOG + Settings.PERFORMANCELOG_QUERY_METHOD + query + " Total query method" + Settings.PERFORMANCELOG_DURATION + (end - start) / 1000); + IOUtils.closeQuietly(out); + return; + } else { + MNodeService mnode = MNodeService.getInstance(request); + mnode.setSession(session); + QueryEngineDescription qed = mnode.getQueryEngineDescription(session, engine); + response.setContentType("text/xml"); + response.setStatus(200); + out = response.getOutputStream(); + TypeMarshaller.marshalTypeToOutputStream(qed, out); + IOUtils.closeQuietly(out); + return; + } + } + + + } catch (BaseException be) { + // report Exceptions as clearly as possible try { out = response.getOutputStream(); } catch (IOException e) { logMetacat.error("Could not get output stream from response", e); } - serializeException(be, out); - } catch (Exception e) { - // report Exceptions as clearly and generically as possible - logMetacat.error(e.getClass() + ": " + e.getMessage(), e); + serializeException(be, out); + } catch (Exception e) { + // report Exceptions as clearly and generically as possible + logMetacat.error(e.getClass() + ": " + e.getMessage(), e); try { out = response.getOutputStream(); } catch (IOException ioe) { logMetacat.error("Could not get output stream from response", ioe); } ServiceFailure se = new ServiceFailure("0000", e.getMessage()); - serializeException(se, out); - } - } - - /* - * Handle the solr query sent by the http post method - */ - private void doPostQuery(String engine) { - OutputStream out = null; - try { - // NOTE: we set the session explicitly for the MNode instance since these methods do not provide a parameter - collectMultipartParams(); - MNodeService mnode = MNodeService.getInstance(request); - if(multipartparams == null || multipartparams.isEmpty()) { - throw new InvalidRequest("2823", "The request doesn't have any query information by the HTTP POST method."); - } - HashMap params = new HashMap(); - for(String key : multipartparams.keySet()) { - List values = multipartparams.get(key); - logMetacat.debug("MNResourceHandler.doPostQuery -the key "+key +" has the value "+values); - if(values != null) { - String[] arrayValues = values.toArray(new String[0]); - params.put(key, arrayValues); - } - } - mnode.setSession(session); - InputStream stream = mnode.postQuery(session, engine, params); - // set the content-type if we have it from the implementation - if (stream instanceof ContentTypeInputStream) { - response.setContentType(((ContentTypeInputStream) stream).getContentType()); - } - response.setStatus(200); - out = response.getOutputStream(); - // write the results to the output stream - IOUtils.copyLarge(stream, out); - IOUtils.closeQuietly(out); - return; - } catch (BaseException be) { - // report Exceptions as clearly as possible - try { - out = response.getOutputStream(); - } catch (IOException e) { - logMetacat.error("Could not get output stream from response", e); - } - serializeException(be, out); - } catch (Exception e) { - // report Exceptions as clearly and generically as possible - logMetacat.error(e.getClass() + ": " + e.getMessage(), e); - try { - out = response.getOutputStream(); - } catch (IOException ioe) { - logMetacat.error("Could not get output stream from response", ioe); - } - ServiceFailure se = new ServiceFailure("0000", e.getMessage()); - serializeException(se, out); - } - } - - private void doViews(String format, String pid) { - + serializeException(se, out); + } + } + + /* + * Handle the solr query sent by the http post method + */ + private void doPostQuery(String engine) { + OutputStream out = null; + try { + // NOTE: we set the session explicitly for the MNode instance since these methods do not provide a parameter + collectMultipartParams(); + MNodeService mnode = MNodeService.getInstance(request); + if (multipartparams == null || multipartparams.isEmpty()) { + throw new InvalidRequest("2823", "The request doesn't have any query information by the HTTP POST method."); + } + HashMap params = new HashMap(); + for (String key : multipartparams.keySet()) { + List values = multipartparams.get(key); + logMetacat.debug("MNResourceHandler.doPostQuery -the key " + key + " has the value " + values); + if (values != null) { + String[] arrayValues = values.toArray(new String[0]); + params.put(key, arrayValues); + } + } + mnode.setSession(session); + InputStream stream = mnode.postQuery(session, engine, params); + // set the content-type if we have it from the implementation + if (stream instanceof ContentTypeInputStream) { + response.setContentType(((ContentTypeInputStream) stream).getContentType()); + } + response.setStatus(200); + out = response.getOutputStream(); + // write the results to the output stream + IOUtils.copyLarge(stream, out); + IOUtils.closeQuietly(out); + return; + } catch (BaseException be) { + // report Exceptions as clearly as possible + try { + out = response.getOutputStream(); + } catch (IOException e) { + logMetacat.error("Could not get output stream from response", e); + } + serializeException(be, out); + } catch (Exception e) { + // report Exceptions as clearly and generically as possible + logMetacat.error(e.getClass() + ": " + e.getMessage(), e); + try { + out = response.getOutputStream(); + } catch (IOException ioe) { + logMetacat.error("Could not get output stream from response", ioe); + } + ServiceFailure se = new ServiceFailure("0000", e.getMessage()); + serializeException(se, out); + } + } + + private void doViews(String format, String pid) { + OutputStream out = null; MNodeService mnode = MNodeService.getInstance(request); - try { - // get a list of views - if (pid != null) { - long start = System.currentTimeMillis(); - Identifier identifier = new Identifier(); - identifier.setValue(pid); - InputStream stream = null; - try { - stream = mnode.view(session, format, identifier); - // set the content-type if we have it from the implementation - if (stream instanceof ContentTypeInputStream) { - response.setContentType(((ContentTypeInputStream) stream).getContentType()); - } - response.setStatus(200); - out = response.getOutputStream(); - // write the results to the output stream - IOUtils.copyLarge(stream, out); - } finally { - if (stream != null) { - IOUtils.closeQuietly(stream); - } - } - long end = System.currentTimeMillis(); - IOUtils.closeQuietly(out); - logMetacat.info(Settings.PERFORMANCELOG + pid + Settings.PERFORMANCELOG_VIEW_METHOD + " Total view method" + Settings.PERFORMANCELOG_DURATION + (end-start)/1000); - return; - } else { - // TODO: list the registered views - //BaseException ni = new NotImplemented("9999", "MN.listViews() is not implemented at this node"); + try { + // get a list of views + if (pid != null) { + long start = System.currentTimeMillis(); + Identifier identifier = new Identifier(); + identifier.setValue(pid); + InputStream stream = null; + try { + stream = mnode.view(session, format, identifier); + // set the content-type if we have it from the implementation + if (stream instanceof ContentTypeInputStream) { + response.setContentType(((ContentTypeInputStream) stream).getContentType()); + } + response.setStatus(200); + out = response.getOutputStream(); + // write the results to the output stream + IOUtils.copyLarge(stream, out); + } finally { + if (stream != null) { + IOUtils.closeQuietly(stream); + } + } + long end = System.currentTimeMillis(); + IOUtils.closeQuietly(out); + logMetacat.info(Settings.PERFORMANCELOG + pid + Settings.PERFORMANCELOG_VIEW_METHOD + " Total view method" + Settings.PERFORMANCELOG_DURATION + (end - start) / 1000); + return; + } else { + // TODO: list the registered views + //BaseException ni = new NotImplemented("9999", "MN.listViews() is not implemented at this node"); //throw ni; - OptionList list = mnode.listViews(session); - - response.setContentType("text/xml"); - response.setStatus(200); - TypeMarshaller.marshalTypeToOutputStream(list, response.getOutputStream()); - IOUtils.closeQuietly(response.getOutputStream()); - } - - - } catch (BaseException be) { - // report Exceptions as clearly as possible + OptionList list = mnode.listViews(session); + + response.setContentType("text/xml"); + response.setStatus(200); + TypeMarshaller.marshalTypeToOutputStream(list, response.getOutputStream()); + IOUtils.closeQuietly(response.getOutputStream()); + } + + + } catch (BaseException be) { + // report Exceptions as clearly as possible try { out = response.getOutputStream(); } catch (IOException e) { logMetacat.error("Could not get output stream from response", e); } - serializeException(be, out); - } catch (Exception e) { - // report Exceptions as clearly and generically as possible - logMetacat.error(e.getClass() + ": " + e.getMessage(), e); + serializeException(be, out); + } catch (Exception e) { + // report Exceptions as clearly and generically as possible + logMetacat.error(e.getClass() + ": " + e.getMessage(), e); try { out = response.getOutputStream(); } catch (IOException ioe) { logMetacat.error("Could not get output stream from response", ioe); } ServiceFailure se = new ServiceFailure("0000", e.getMessage()); - serializeException(se, out); - } - } - - /** - * Handles notification of system metadata changes for the given identifier - * - * @param id the identifier for the object - * @throws InvalidToken - * @throws InvalidRequest - * @throws NotAuthorized - * @throws ServiceFailure - * @throws NotImplemented - */ - private void systemMetadataChanged() - throws NotImplemented, ServiceFailure, NotAuthorized, InvalidRequest, - InvalidToken { - - ReadOnlyChecker checker = new ReadOnlyChecker(); - boolean isReadOnlyMode = checker.isReadOnly(); - if(isReadOnlyMode) { - throw new ServiceFailure("1333", ReadOnlyChecker.DATAONEERROR); - } - - //final long serialVersion = 0L; - String serialVersionStr = null; - String dateSysMetaLastModifiedStr = null; - - // mkae sure we have the multipart params - try { + serializeException(se, out); + } + } + + /** + * Handles notification of system metadata changes for the given identifier + * + * @param id the identifier for the object + * @throws InvalidToken + * @throws InvalidRequest + * @throws NotAuthorized + * @throws ServiceFailure + * @throws NotImplemented + */ + private void systemMetadataChanged() + throws NotImplemented, ServiceFailure, NotAuthorized, InvalidRequest, + InvalidToken { + + ReadOnlyChecker checker = new ReadOnlyChecker(); + boolean isReadOnlyMode = checker.isReadOnly(); + if (isReadOnlyMode) { + throw new ServiceFailure("1333", ReadOnlyChecker.DATAONEERROR); + } + + //final long serialVersion = 0L; + String serialVersionStr = null; + String dateSysMetaLastModifiedStr = null; + + // mkae sure we have the multipart params + try { initMultipartParams(); } catch (Exception e1) { throw new ServiceFailure("1333", "Could not collect the multipart params for the request"); } - - // get the pid - String id = null; - try { - id = multipartparams.get("pid").get(0); - } catch (NullPointerException e) { - String msg = "The 'pid' must be provided as a parameter and was not."; - logMetacat.error(msg); - throw new InvalidRequest("1334", msg); - } - final Identifier pid = new Identifier(); - pid.setValue(id); - - // get the serialVersion - try { - serialVersionStr = multipartparams.get("serialVersion").get(0); - } catch (NullPointerException e) { - String msg = "The 'serialVersion' must be provided as a parameter and was not."; - logMetacat.error(msg); - throw new InvalidRequest("1334", msg); - - } - - final long serialVersion = (new Long(serialVersionStr)).longValue(); - - // get the dateSysMetaLastModified - try { - dateSysMetaLastModifiedStr = multipartparams.get("dateSysMetaLastModified").get(0); - - - } catch (NullPointerException e) { - String msg = - "The 'dateSysMetaLastModified' must be provided as a " + - "parameter and was not, or was an invalid representation of the timestamp."; - logMetacat.error(msg); - throw new InvalidRequest("1334", msg); - - } - final Date dateSysMetaLastModified = DateTimeMarshaller.deserializeDateToUTC(dateSysMetaLastModifiedStr); - - // check authorization before sending to implementation - D1AuthHelper authDel = new D1AuthHelper(request, pid, "1331", "????"); - authDel.doAdminAuthorization(session); + + // get the pid + String id = null; + try { + id = multipartparams.get("pid").get(0); + } catch (NullPointerException e) { + String msg = "The 'pid' must be provided as a parameter and was not."; + logMetacat.error(msg); + throw new InvalidRequest("1334", msg); + } + final Identifier pid = new Identifier(); + pid.setValue(id); + + // get the serialVersion + try { + serialVersionStr = multipartparams.get("serialVersion").get(0); + } catch (NullPointerException e) { + String msg = "The 'serialVersion' must be provided as a parameter and was not."; + logMetacat.error(msg); + throw new InvalidRequest("1334", msg); + + } + + final long serialVersion = (new Long(serialVersionStr)).longValue(); + + // get the dateSysMetaLastModified + try { + dateSysMetaLastModifiedStr = multipartparams.get("dateSysMetaLastModified").get(0); + + + } catch (NullPointerException e) { + String msg = + "The 'dateSysMetaLastModified' must be provided as a " + + "parameter and was not, or was an invalid representation of the timestamp."; + logMetacat.error(msg); + throw new InvalidRequest("1334", msg); + + } + final Date dateSysMetaLastModified = DateTimeMarshaller.deserializeDateToUTC(dateSysMetaLastModifiedStr); + + // check authorization before sending to implementation + D1AuthHelper authDel = new D1AuthHelper(request, pid, "1331", "????"); + authDel.doAdminAuthorization(session); // boolean authorized = MNodeService.getInstance(request).isAdminAuthorized(session); // if (!authorized) { // String msg = "User is not authorized to call systemMetadataChanged"; // NotAuthorized failure = new NotAuthorized("1331", msg); // throw failure; // } - - // run it in a thread to avoid connection timeout - final String ipAddress = request.getRemoteAddr(); - final String userAgent = request.getHeader("User-Agent"); - Runnable runner = new Runnable() { - @Override - public void run() { - try { - // call the service - MNodeService.getInstance(request, ipAddress, userAgent).systemMetadataChanged(session, pid, serialVersion, dateSysMetaLastModified); - } catch (Exception e) { - logMetacat.error("Error running replication: " + e.getMessage(), e); - throw new RuntimeException(e.getMessage(), e); - } - } - }; - // submit the task, and that's it - executor.submit(runner); - - // thread was started, so we return success - response.setStatus(200); - } - - /** - * Handles identifier generation calls - * - * @throws InvalidRequest - * @throws NotImplemented - * @throws NotAuthorized - * @throws ServiceFailure - * @throws InvalidToken - * @throws IOException - * @throws MarshallingException - */ - private void generateIdentifier() throws InvalidToken, ServiceFailure, NotAuthorized, NotImplemented, InvalidRequest, IOException, MarshallingException { - - // make sure we have the multipart params - try { + + // run it in a thread to avoid connection timeout + final String ipAddress = request.getRemoteAddr(); + final String userAgent = request.getHeader("User-Agent"); + Runnable runner = new Runnable() { + @Override + public void run() { + try { + // call the service + MNodeService.getInstance(request, ipAddress, userAgent).systemMetadataChanged(session, pid, serialVersion, dateSysMetaLastModified); + } catch (Exception e) { + logMetacat.error("Error running replication: " + e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); + } + } + }; + // submit the task, and that's it + executor.submit(runner); + + // thread was started, so we return success + response.setStatus(200); + } + + /** + * Handles identifier generation calls + * + * @throws InvalidRequest + * @throws NotImplemented + * @throws NotAuthorized + * @throws ServiceFailure + * @throws InvalidToken + * @throws IOException + * @throws MarshallingException + */ + private void generateIdentifier() throws InvalidToken, ServiceFailure, NotAuthorized, NotImplemented, InvalidRequest, IOException, MarshallingException { + + // make sure we have the multipart params + try { initMultipartParams(); } catch (Exception e1) { throw new ServiceFailure("1333", "Could not collect the multipart params for the request"); } - - // get the scheme + + // get the scheme String scheme = null; - try { - scheme = multipartparams.get("scheme").get(0); - } catch (NullPointerException e) { - String msg = "The 'scheme' parameter was not provided, using default"; - logMetacat.warn(msg); - } - - // get the fragment + try { + scheme = multipartparams.get("scheme").get(0); + } catch (NullPointerException e) { + String msg = "The 'scheme' parameter was not provided, using default"; + logMetacat.warn(msg); + } + + // get the fragment String fragment = null; - try { - fragment = multipartparams.get("fragment").get(0); - } catch (NullPointerException e) { - String msg = "The 'fragment' parameter was not provided, using default"; - logMetacat.warn(msg); - } + try { + fragment = multipartparams.get("fragment").get(0); + } catch (NullPointerException e) { + String msg = "The 'fragment' parameter was not provided, using default"; + logMetacat.warn(msg); + } - // call the service - Identifier identifier = MNodeService.getInstance(request).generateIdentifier(session, scheme, fragment); - response.setStatus(200); - response.setContentType("text/xml"); - OutputStream out = response.getOutputStream(); - TypeMarshaller.marshalTypeToOutputStream(identifier, out); - IOUtils.closeQuietly(out); - } + // call the service + Identifier identifier = MNodeService.getInstance(request).generateIdentifier(session, scheme, fragment); + response.setStatus(200); + response.setContentType("text/xml"); + OutputStream out = response.getOutputStream(); + TypeMarshaller.marshalTypeToOutputStream(identifier, out); + IOUtils.closeQuietly(out); + } - /** - * Checks the access policy - * @param id - * @return - * @throws ServiceFailure - * @throws InvalidToken - * @throws NotFound - * @throws NotAuthorized - * @throws NotImplemented - * @throws InvalidRequest - */ - private boolean isAuthorized(String id) throws ServiceFailure, InvalidToken, NotFound, NotAuthorized, NotImplemented, InvalidRequest { + /** + * Checks the access policy + * + * @param id + * @return + * @throws ServiceFailure + * @throws InvalidToken + * @throws NotFound + * @throws NotAuthorized + * @throws NotImplemented + * @throws InvalidRequest + */ + private boolean isAuthorized(String id) throws ServiceFailure, InvalidToken, NotFound, NotAuthorized, NotImplemented, InvalidRequest { Identifier pid = new Identifier(); pid.setValue(id); Permission permission = null; @@ -917,10 +916,10 @@ private boolean isAuthorized(String id) throws ServiceFailure, InvalidToken, Not response.setStatus(200); response.setContentType("text/xml"); return result; - } - - private void getToken() throws Exception { - + } + + private void getToken() throws Exception { + if (this.session != null) { String userId = this.session.getSubject().getValue(); String fullName = null; @@ -930,27 +929,27 @@ private void getToken() throws Exception { } catch (Exception e) { logMetacat.warn(e.getMessage(), e); } - + String token = null; token = TokenGenerator.getInstance().getJWT(userId, fullName); - + response.setStatus(200); response.setContentType("text/plain"); - OutputStream out = response.getOutputStream(); - out.write(token.getBytes(MetaCatServlet.DEFAULT_ENCODING)); - out.close(); + OutputStream out = response.getOutputStream(); + out.write(token.getBytes(MetaCatServlet.DEFAULT_ENCODING)); + out.close(); } else { response.setStatus(401); response.setContentType("text/plain"); OutputStream out = response.getOutputStream(); - out.write("No session information found".getBytes(MetaCatServlet.DEFAULT_ENCODING)); - out.close(); + out.write("No session information found".getBytes(MetaCatServlet.DEFAULT_ENCODING)); + out.close(); } - - } - - private void whoami() throws Exception { - + + } + + private void whoami() throws Exception { + if (this.session != null) { Subject subject = this.session.getSubject(); SubjectInfo subjectInfo = null; @@ -959,55 +958,56 @@ private void whoami() throws Exception { } catch (Exception e) { logMetacat.warn(e.getMessage(), e); } - + response.setStatus(200); response.setContentType("text/plain"); - OutputStream out = response.getOutputStream(); - - if (subjectInfo != null) { - TypeMarshaller.marshalTypeToOutputStream(subjectInfo, out); - } else { - TypeMarshaller.marshalTypeToOutputStream(subject, out); - } - - out.close(); + OutputStream out = response.getOutputStream(); + + if (subjectInfo != null) { + TypeMarshaller.marshalTypeToOutputStream(subjectInfo, out); + } else { + TypeMarshaller.marshalTypeToOutputStream(subject, out); + } + + out.close(); } else { response.setStatus(401); response.setContentType("text/plain"); OutputStream out = response.getOutputStream(); - out.write("No session information found".getBytes(MetaCatServlet.DEFAULT_ENCODING)); - out.close(); + out.write("No session information found".getBytes(MetaCatServlet.DEFAULT_ENCODING)); + out.close(); } - - } - - /** - * Get the status of the system. Now we only support to get the size of the index queue. - */ - private void getStatus() throws IOException, NotAuthorized, ServiceFailure { - InputStream result = MNodeService.getInstance(request).getStatus(this.session); - response.setStatus(200); - response.setContentType("text/xml"); - OutputStream out = response.getOutputStream(); - out = response.getOutputStream(); - IOUtils.copy(result, out); - IOUtils.closeQuietly(out); - //IOUtils.copyLarge(result, out); - } - - /** - * Processes failed synchronization message - * @throws NotImplemented - * @throws ServiceFailure - * @throws NotAuthorized - * @throws InvalidRequest - * @throws MarshallingException - * @throws IllegalAccessException - * @throws InstantiationException - * @throws IOException - */ - private void syncError() throws NotImplemented, ServiceFailure, NotAuthorized, InvalidRequest, MarshallingException, IOException, InstantiationException, IllegalAccessException { - SynchronizationFailed syncFailed = null; + + } + + /** + * Get the status of the system. Now we only support to get the size of the index queue. + */ + private void getStatus() throws IOException, NotAuthorized, ServiceFailure { + InputStream result = MNodeService.getInstance(request).getStatus(this.session); + response.setStatus(200); + response.setContentType("text/xml"); + OutputStream out = response.getOutputStream(); + out = response.getOutputStream(); + IOUtils.copy(result, out); + IOUtils.closeQuietly(out); + //IOUtils.copyLarge(result, out); + } + + /** + * Processes failed synchronization message + * + * @throws NotImplemented + * @throws ServiceFailure + * @throws NotAuthorized + * @throws InvalidRequest + * @throws MarshallingException + * @throws IllegalAccessException + * @throws InstantiationException + * @throws IOException + */ + private void syncError() throws NotImplemented, ServiceFailure, NotAuthorized, InvalidRequest, MarshallingException, IOException, InstantiationException, IllegalAccessException { + SynchronizationFailed syncFailed = null; try { syncFailed = collectSynchronizationFailed(); } catch (ParserConfigurationException e) { @@ -1015,548 +1015,516 @@ private void syncError() throws NotImplemented, ServiceFailure, NotAuthorized, I } catch (SAXException e) { throw new ServiceFailure("2161", e.getMessage()); } - + MNodeService.getInstance(request).synchronizationFailed(session, syncFailed); - } + } /** - * Calculate the checksum - * @throws NotImplemented - * @throws MarshallingException - * @throws IOException - * @throws InvalidToken - * @throws ServiceFailure - * @throws NotAuthorized - * @throws NotFound - * @throws InvalidRequest - */ - private void checksum(String pid) throws NotImplemented, MarshallingException, IOException, InvalidToken, ServiceFailure, NotAuthorized, NotFound, InvalidRequest { - String checksumAlgorithm = "MD5"; - try { - checksumAlgorithm = PropertyService.getProperty("dataone.checksumAlgorithm.default"); - } catch(Exception e) { - logMetacat.warn("Could not lookup configured default checksum algorithm, using: " + checksumAlgorithm); - } - - Identifier pidid = new Identifier(); - pidid.setValue(pid); - try { - checksumAlgorithm = params.get("checksumAlgorithm")[0]; - } catch(Exception e) { - //do nothing. use the default - logMetacat.warn("No algorithm specified, using default: " + checksumAlgorithm); - } - logMetacat.debug("getting checksum for object " + pid + " with algorithm " + checksumAlgorithm); - - Checksum c = MNodeService.getInstance(request).getChecksum(session, pidid, checksumAlgorithm); - logMetacat.debug("got checksum " + c.getValue()); - response.setStatus(200); - logMetacat.debug("serializing response"); - TypeMarshaller.marshalTypeToOutputStream(c, response.getOutputStream()); - logMetacat.debug("done serializing response."); - IOUtils.closeQuietly(response.getOutputStream()); - - } - + * Calculate the checksum + * + * @throws NotImplemented + * @throws MarshallingException + * @throws IOException + * @throws InvalidToken + * @throws ServiceFailure + * @throws NotAuthorized + * @throws NotFound + * @throws InvalidRequest + */ + private void checksum(String pid) throws NotImplemented, MarshallingException, IOException, InvalidToken, ServiceFailure, NotAuthorized, NotFound, InvalidRequest { + String checksumAlgorithm = "MD5"; + try { + checksumAlgorithm = PropertyService.getProperty("dataone.checksumAlgorithm.default"); + } catch (Exception e) { + logMetacat.warn("Could not lookup configured default checksum algorithm, using: " + checksumAlgorithm); + } + + Identifier pidid = new Identifier(); + pidid.setValue(pid); + try { + checksumAlgorithm = params.get("checksumAlgorithm")[0]; + } catch (Exception e) { + //do nothing. use the default + logMetacat.warn("No algorithm specified, using default: " + checksumAlgorithm); + } + logMetacat.debug("getting checksum for object " + pid + " with algorithm " + checksumAlgorithm); + + Checksum c = MNodeService.getInstance(request).getChecksum(session, pidid, checksumAlgorithm); + logMetacat.debug("got checksum " + c.getValue()); + response.setStatus(200); + logMetacat.debug("serializing response"); + TypeMarshaller.marshalTypeToOutputStream(c, response.getOutputStream()); + logMetacat.debug("done serializing response."); + IOUtils.closeQuietly(response.getOutputStream()); + + } + /** - * handle the replicate action for MN - * @throws MarshallingException - * @throws FileUploadException - * @throws IOException - * @throws InvalidRequest - * @throws ServiceFailure - * @throws UnsupportedType - * @throws InsufficientResources - * @throws NotAuthorized - * @throws NotImplemented - * @throws IllegalAccessException - * @throws InstantiationException - * @throws InvalidToken - */ - private void replicate() - throws ServiceFailure, InvalidRequest, IOException, FileUploadException, - MarshallingException, NotImplemented, NotAuthorized, InsufficientResources, - UnsupportedType, InstantiationException, IllegalAccessException, InvalidToken { - - logMetacat.debug("in POST replicate()"); - ReadOnlyChecker checker = new ReadOnlyChecker(); - boolean isReadOnlyMode = checker.isReadOnly(); - if(isReadOnlyMode) { - throw new ServiceFailure("2151", ReadOnlyChecker.DATAONEERROR); - } - - // somewhat unorthodox, but the call is asynchronous and we'd like to return this info sooner - boolean allowed = false; - if (session == null) { - String msg = "No session was provided."; - NotAuthorized failure = new NotAuthorized("2152", msg); - throw failure; - } else { - // TODO: should we refactore replicate() in MNodeservice to not replicate, it would avoid a possible second listNodes call... - D1AuthHelper authDel = new D1AuthHelper(request, null, "2152", "????"); - authDel.doAdminAuthorization(session); + * handle the replicate action for MN + * + * @throws MarshallingException + * @throws FileUploadException + * @throws IOException + * @throws InvalidRequest + * @throws ServiceFailure + * @throws UnsupportedType + * @throws InsufficientResources + * @throws NotAuthorized + * @throws NotImplemented + * @throws IllegalAccessException + * @throws InstantiationException + * @throws InvalidToken + */ + private void replicate() + throws ServiceFailure, InvalidRequest, IOException, FileUploadException, + MarshallingException, NotImplemented, NotAuthorized, InsufficientResources, + UnsupportedType, InstantiationException, IllegalAccessException, InvalidToken { + + logMetacat.debug("in POST replicate()"); + ReadOnlyChecker checker = new ReadOnlyChecker(); + boolean isReadOnlyMode = checker.isReadOnly(); + if (isReadOnlyMode) { + throw new ServiceFailure("2151", ReadOnlyChecker.DATAONEERROR); + } + + // somewhat unorthodox, but the call is asynchronous and we'd like to return this info sooner + boolean allowed = false; + if (session == null) { + String msg = "No session was provided."; + NotAuthorized failure = new NotAuthorized("2152", msg); + throw failure; + } else { + // TODO: should we refactore replicate() in MNodeservice to not replicate, it would avoid a possible second listNodes call... + D1AuthHelper authDel = new D1AuthHelper(request, null, "2152", "????"); + authDel.doAdminAuthorization(session); // allowed = MNodeService.getInstance(request).isAdminAuthorized(session); // if (!allowed) { // String msg = "User is not an admin user"; // NotAuthorized failure = new NotAuthorized("2152", msg); // throw failure; // } - } - - // parse the systemMetadata - Map files = collectMultipartFiles(); - final SystemMetadata sysmeta = TypeMarshaller.unmarshalTypeFromFile(SystemMetadata.class, files.get("sysmeta")); - - String sn = multipartparams.get("sourceNode").get(0); - logMetacat.debug("sourceNode: " + sn); - final NodeReference sourceNode = new NodeReference(); - sourceNode.setValue(sn); - - // run it in a thread to avoid connection timeout - final String ipAddress = request.getRemoteAddr(); - final String userAgent = request.getHeader("User-Agent"); - Runnable runner = new Runnable() { + } + + // parse the systemMetadata + Map files = collectMultipartFiles(); + final SystemMetadata sysmeta = TypeMarshaller.unmarshalTypeFromFile(SystemMetadata.class, files.get("sysmeta")); + + String sn = multipartparams.get("sourceNode").get(0); + logMetacat.debug("sourceNode: " + sn); + final NodeReference sourceNode = new NodeReference(); + sourceNode.setValue(sn); + + // run it in a thread to avoid connection timeout + final String ipAddress = request.getRemoteAddr(); + final String userAgent = request.getHeader("User-Agent"); + Runnable runner = new Runnable() { @Override public void run() { try { - MNodeService.getInstance(request, ipAddress, userAgent).replicate(session, sysmeta, sourceNode); + MNodeService.getInstance(request, ipAddress, userAgent).replicate(session, sysmeta, sourceNode); } catch (Exception e) { logMetacat.error("Error running replication: " + e.getMessage(), e); throw new RuntimeException(e.getMessage(), e); } } - }; - // submit the task, and that's it - executor.submit(runner); - - // thread was started, so we return success - response.setStatus(200); - - } + }; + // submit the task, and that's it + executor.submit(runner); - /** - * Handle the getReplica action for the MN - * @param id the identifier for the object - * @throws NotFound - * @throws ServiceFailure - * @throws NotImplemented - * @throws NotAuthorized - * @throws InvalidToken - * @throws InvalidRequest - */ - private void getReplica(String id) - throws InvalidRequest, InvalidToken, NotAuthorized, NotImplemented, - ServiceFailure, NotFound { - - Identifier pid = new Identifier(); - pid.setValue(id); - OutputStream out = null; - InputStream dataBytes = null; - - try { - // call the service - dataBytes = MNodeService.getInstance(request).getReplica(session, pid); - - response.setContentType("application/octet-stream"); - response.setStatus(200); - out = response.getOutputStream(); - // write the object to the output stream - IOUtils.copyLarge(dataBytes, out); - IOUtils.closeQuietly(out); - } catch (IOException e) { - String msg = "There was an error writing the output: " + e.getMessage(); - logMetacat.error(msg); - throw new ServiceFailure("2181", msg); - - } + // thread was started, so we return success + response.setStatus(200); - } + } - /** - * Get the Node information - * - * @throws MarshallingException - * @throws IOException - * @throws InvalidRequest - * @throws ServiceFailure - * @throws NotAuthorized - * @throws NotImplemented - */ - private void node() - throws MarshallingException, IOException, NotImplemented, NotAuthorized, ServiceFailure, InvalidRequest { - - Node n = MNodeService.getInstance(request).getCapabilities(); - - response.setContentType("text/xml"); - response.setStatus(200); - TypeMarshaller.marshalTypeToOutputStream(n, response.getOutputStream()); - IOUtils.closeQuietly(response.getOutputStream()); - - } - - /** - * MN_crud.describe() - * http://mule1.dataone.org/ArchitectureDocs/mn_api_crud.html#MN_crud.describe - * @param pid - * @throws InvalidRequest - * @throws NotImplemented - * @throws NotFound - * @throws NotAuthorized - * @throws ServiceFailure - * @throws InvalidToken - */ - private void describeObject(String pid) throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented, InvalidRequest - { - - response.setContentType("text/xml"); + /** + * Handle the getReplica action for the MN + * + * @param id the identifier for the object + * @throws NotFound + * @throws ServiceFailure + * @throws NotImplemented + * @throws NotAuthorized + * @throws InvalidToken + * @throws InvalidRequest + */ + private void getReplica(String id) + throws InvalidRequest, InvalidToken, NotAuthorized, NotImplemented, + ServiceFailure, NotFound { - Identifier id = new Identifier(); - id.setValue(pid); + Identifier pid = new Identifier(); + pid.setValue(id); + OutputStream out = null; + InputStream dataBytes = null; + + try { + // call the service + dataBytes = MNodeService.getInstance(request).getReplica(session, pid); + + response.setContentType("application/octet-stream"); + response.setStatus(200); + out = response.getOutputStream(); + // write the object to the output stream + IOUtils.copyLarge(dataBytes, out); + IOUtils.closeQuietly(out); + } catch (IOException e) { + String msg = "There was an error writing the output: " + e.getMessage(); + logMetacat.error(msg); + throw new ServiceFailure("2181", msg); - DescribeResponse dr = null; - try { - dr = MNodeService.getInstance(request).describe(session, id); - } catch (BaseException e) { - response.setStatus(e.getCode()); - response.addHeader("DataONE-Exception-Name", e.getClass().getName()); - response.addHeader("DataONE-Exception-DetailCode", e.getDetail_code()); - response.addHeader("DataONE-Exception-Description", e.getDescription()); - response.addHeader("DataONE-Exception-PID", id.getValue()); - return; } - - response.setStatus(200); - - //response.addHeader("pid", pid); - response.addHeader("DataONE-Checksum", dr.getDataONE_Checksum().getAlgorithm() + "," + dr.getDataONE_Checksum().getValue()); - response.addHeader("Content-Length", dr.getContent_Length() + ""); - response.addHeader("Last-Modified", DateTimeMarshaller.serializeDateToUTC(dr.getLast_Modified())); - response.addHeader("DataONE-ObjectFormat", dr.getDataONE_ObjectFormatIdentifier().getValue()); - response.addHeader("DataONE-SerialVersion", dr.getSerialVersion().toString()); - - } - - /** - * get the logs based on passed params. Available - * See http://mule1.dataone.org/ArchitectureDocs/mn_api_crud.html#MN_crud.getLogRecords - * for more info - * @throws NotImplemented - * @throws InvalidRequest - * @throws NotAuthorized - * @throws ServiceFailure - * @throws InvalidToken - * @throws IOException - * @throws MarshallingException - */ - private void getLog() throws InvalidToken, ServiceFailure, NotAuthorized, InvalidRequest, NotImplemented, IOException, MarshallingException - { - - Date fromDate = null; - Date toDate = null; - String event = null; - Integer start = null; - Integer count = null; - String pidFilter = null; - - try { - String fromDateS = params.get("fromDate")[0]; - logMetacat.debug("param fromDateS: " + fromDateS); - fromDate = DateTimeMarshaller.deserializeDateToUTC(fromDateS); - } catch (Exception e) { - logMetacat.warn("Could not parse fromDate: " + e.getMessage()); - } - try { - String toDateS = params.get("toDate")[0]; - logMetacat.debug("param toDateS: " + toDateS); - toDate = DateTimeMarshaller.deserializeDateToUTC(toDateS); - } catch (Exception e) { - logMetacat.warn("Could not parse toDate: " + e.getMessage()); + } + + /** + * Get the Node information + * + * @throws MarshallingException + * @throws IOException + * @throws InvalidRequest + * @throws ServiceFailure + * @throws NotAuthorized + * @throws NotImplemented + */ + private void node() + throws MarshallingException, IOException, NotImplemented, NotAuthorized, ServiceFailure, InvalidRequest { + + Node n = MNodeService.getInstance(request).getCapabilities(); + + response.setContentType("text/xml"); + response.setStatus(200); + TypeMarshaller.marshalTypeToOutputStream(n, response.getOutputStream()); + IOUtils.closeQuietly(response.getOutputStream()); + + } + + /** + * MN_crud.describe() + * http://mule1.dataone.org/ArchitectureDocs/mn_api_crud.html#MN_crud.describe + * + * @param pid + * @throws InvalidRequest + * @throws NotImplemented + * @throws NotFound + * @throws NotAuthorized + * @throws ServiceFailure + * @throws InvalidToken + */ + private void describeObject(String pid) throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented, InvalidRequest { + + response.setContentType("text/xml"); + + Identifier id = new Identifier(); + id.setValue(pid); + + DescribeResponse dr = null; + try { + dr = MNodeService.getInstance(request).describe(session, id); + } catch (BaseException e) { + response.setStatus(e.getCode()); + response.addHeader("DataONE-Exception-Name", e.getClass().getName()); + response.addHeader("DataONE-Exception-DetailCode", e.getDetail_code()); + response.addHeader("DataONE-Exception-Description", e.getDescription()); + response.addHeader("DataONE-Exception-PID", id.getValue()); + return; } - try { - event = params.get("event")[0]; - } catch (Exception e) { - logMetacat.warn("Could not parse event: " + e.getMessage()); + + response.setStatus(200); + + //response.addHeader("pid", pid); + response.addHeader("DataONE-Checksum", dr.getDataONE_Checksum().getAlgorithm() + "," + dr.getDataONE_Checksum().getValue()); + response.addHeader("Content-Length", dr.getContent_Length() + ""); + response.addHeader("Last-Modified", DateTimeMarshaller.serializeDateToUTC(dr.getLast_Modified())); + response.addHeader("DataONE-ObjectFormat", dr.getDataONE_ObjectFormatIdentifier().getValue()); + response.addHeader("DataONE-SerialVersion", dr.getSerialVersion().toString()); + + + } + + /** + * get the logs based on passed params. Available + * See http://mule1.dataone.org/ArchitectureDocs/mn_api_crud.html#MN_crud.getLogRecords + * for more info + * + * @throws NotImplemented + * @throws InvalidRequest + * @throws NotAuthorized + * @throws ServiceFailure + * @throws InvalidToken + * @throws IOException + * @throws MarshallingException + */ + private void getLog() throws InvalidToken, ServiceFailure, NotAuthorized, InvalidRequest, NotImplemented, IOException, MarshallingException { + + Date fromDate = null; + Date toDate = null; + String event = null; + Integer start = null; + Integer count = null; + String pidFilter = null; + + try { + String fromDateS = params.get("fromDate")[0]; + logMetacat.debug("param fromDateS: " + fromDateS); + fromDate = DateTimeMarshaller.deserializeDateToUTC(fromDateS); + } catch (Exception e) { + logMetacat.warn("Could not parse fromDate: " + e.getMessage()); } - logMetacat.debug("fromDate: " + fromDate + " toDate: " + toDate); - - try { - start = Integer.parseInt(params.get("start")[0]); - } catch (Exception e) { + try { + String toDateS = params.get("toDate")[0]; + logMetacat.debug("param toDateS: " + toDateS); + toDate = DateTimeMarshaller.deserializeDateToUTC(toDateS); + } catch (Exception e) { + logMetacat.warn("Could not parse toDate: " + e.getMessage()); + } + try { + event = params.get("event")[0]; + } catch (Exception e) { + logMetacat.warn("Could not parse event: " + e.getMessage()); + } + logMetacat.debug("fromDate: " + fromDate + " toDate: " + toDate); + + try { + start = Integer.parseInt(params.get("start")[0]); + } catch (Exception e) { logMetacat.warn("Could not parse start: " + e.getMessage()); } - try { - count = Integer.parseInt(params.get("count")[0]); - } catch (Exception e) { + try { + count = Integer.parseInt(params.get("count")[0]); + } catch (Exception e) { logMetacat.warn("Could not parse count: " + e.getMessage()); } - - try { - pidFilter = params.get("idFilter")[0]; - } catch (Exception e) { - logMetacat.warn("Could not parse pidFilter: " + e.getMessage()); - } - - logMetacat.debug("calling getLogRecords"); - Log log = MNodeService.getInstance(request).getLogRecords(session, fromDate, toDate, event, pidFilter, start, count); - - OutputStream out = response.getOutputStream(); - response.setStatus(200); - response.setContentType("text/xml"); - - TypeMarshaller.marshalTypeToOutputStream(log, out); - IOUtils.closeQuietly(out); - } - - - - /** - * Implements REST version of DataONE CRUD API --> get - * @param pid ID of data object to be read - * @throws NotImplemented - * @throws InvalidRequest - * @throws NotFound - * @throws NotAuthorized - * @throws ServiceFailure - * @throws InvalidToken - * @throws IOException - * @throws MarshallingException - */ - protected void getObject(String pid) throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, InvalidRequest, NotImplemented, IOException, MarshallingException { - OutputStream out = null; - - if (pid != null) { //get a specific document - long start = System.currentTimeMillis(); - Identifier id = new Identifier(); - id.setValue(pid); - - SystemMetadata sm = MNodeService.getInstance(request).getSystemMetadata(session, id); - - // set the headers for the content - String mimeType = null; - String charset = null; - ObjectFormat objectFormat = null; - - try { - objectFormat = ObjectFormatCache.getInstance().getFormat(sm.getFormatId()); - } catch (BaseException be) { - logMetacat.warn("Could not lookup ObjectFormat for: " + sm.getFormatId(), be); - } - // do we have mediaType/encoding in SM? - MediaType mediaType = sm.getMediaType(); - if (mediaType == null && objectFormat != null) { - try { - mediaType = objectFormat.getMediaType(); - } catch (Exception e) { - logMetacat.warn("Could not lookup MediaType for: " + sm.getFormatId(), e); - } - } - if (mediaType != null) { - mimeType = mediaType.getName(); - if (mediaType.getPropertyList() != null) { - Iterator iter = mediaType.getPropertyList().iterator(); - while (iter.hasNext()) { - MediaTypeProperty mtp = iter.next(); - if (mtp.getName().equalsIgnoreCase("charset")) { - charset = mtp.getValue(); - mimeType += "; charset=" + charset; - break; - } - } - } - } - // check object format - - // use the fallback from v1 impl - if (mimeType == null) { - mimeType = ObjectFormatInfo.instance().getMimeType(sm.getFormatId().getValue()); - - // still null? - if (mimeType == null) { - mimeType = "application/octet-stream"; - } - } - - // check for filename in SM first - String filename = sm.getFileName(); - // then fallback to using id and extension - if (filename == null) { - String extension = null; - if(objectFormat != null) { - extension = objectFormat.getExtension(); - } - if (extension == null) { - extension = ObjectFormatInfo.instance().getExtension(sm.getFormatId().getValue()); - } - filename = id.getValue(); - if (extension != null && filename != null && !filename.endsWith(extension)) { - filename = id.getValue() + "." + extension; - } - } - response.setContentType(mimeType); - response.setHeader("Content-Disposition", "inline; filename=\"" + filename+"\""); - InputStream data = null; - try { - data = MNodeService.getInstance(request).get(session, id); - out = response.getOutputStream(); - response.setStatus(200); - IOUtils.copyLarge(data, out); - IOUtils.closeQuietly(out); - } finally { - if (data != null) { - IOUtils.closeQuietly(data); - } - } - long end = System.currentTimeMillis(); - logMetacat.info(Settings.PERFORMANCELOG + pid + Settings.PERFORMANCELOG_GET_METHOD + " Total get method" + Settings.PERFORMANCELOG_DURATION + (end-start)/1000); - } - else - { //call listObjects with specified params - Date startTime = null; - Date endTime = null; - ObjectFormatIdentifier formatId = null; - Identifier identifier = null; - boolean replicaStatus = true; - int start = 0; - //TODO: make the max count into a const - int count = 1000; - Enumeration paramlist = request.getParameterNames(); - while (paramlist.hasMoreElements()) - { //parse the params and make the crud call - String name = (String) paramlist.nextElement(); - String[] value = (String[])request.getParameterValues(name); - - if (name.equals("fromDate") && value != null) - { - try - { - //startTime = dateFormat.parse(value[0]); - startTime = DateTimeMarshaller.deserializeDateToUTC(value[0]); - //startTime = parseDateAndConvertToGMT(value[0]); - } - catch(Exception e) - { //if we can't parse it, just don't use the fromDate param - logMetacat.warn("Could not parse fromDate: " + value[0], e); - throw new InvalidRequest("1540", "Could not parse fromDate: " + value[0]+" since "+e.getMessage()); - //startTime = null; - } - } - else if(name.equals("toDate") && value != null) - { - try - { - endTime = DateTimeMarshaller.deserializeDateToUTC(value[0]); - } - catch(Exception e) - { //if we can't parse it, just don't use the toDate param - logMetacat.warn("Could not parse toDate: " + value[0], e); - throw new InvalidRequest("1540", "Could not parse toDate: " + value[0]+" since "+e.getMessage()); - //endTime = null; - } - } - else if(name.equals("formatId") && value != null) - { - formatId = new ObjectFormatIdentifier(); - formatId.setValue(value[0]); - } - else if(name.equals("identifier") && value != null) - { - identifier = new Identifier(); - identifier.setValue(value[0]); - } - else if(name.equals("replicaStatus") && value != null) - { - if(value != null && - value.length > 0 && - (value[0].equalsIgnoreCase("false") || value[0].equalsIgnoreCase("no"))) - { - replicaStatus = false; - } - } - else if(name.equals("start") && value != null) - { - start = new Integer(value[0]).intValue(); - } - else if(name.equals("count") && value != null) - { - count = new Integer(value[0]).intValue(); - } - } - //make the crud call - logMetacat.debug("session: " + session + " startTime: " + startTime + - " endTime: " + endTime + " formatId: " + - formatId + " replicaStatus: " + replicaStatus + - " start: " + start + " count: " + count); - - ObjectList ol = - MNodeService.getInstance(request).listObjects(session, startTime, endTime, - formatId, identifier, replicaStatus, start, count); - - out = response.getOutputStream(); - response.setStatus(200); - response.setContentType("text/xml"); - // Serialize and write it to the output stream - TypeMarshaller.marshalTypeToOutputStream(ol, out); - IOUtils.closeQuietly(out); - } - - } - - /** - * Retrieve data package as Bagit zip - * @param format - * @param pid The pid of the resource map defining the pacakage - * @throws NotImplemented - * @throws NotFound - * @throws NotAuthorized - * @throws ServiceFailure - * @throws InvalidToken - * @throws IOException - * @throws InvalidRequest - */ - protected void getPackage(String format, String pid) throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented, IOException, InvalidRequest { + try { + pidFilter = params.get("idFilter")[0]; + } catch (Exception e) { + logMetacat.warn("Could not parse pidFilter: " + e.getMessage()); + } + + logMetacat.debug("calling getLogRecords"); + Log log = MNodeService.getInstance(request).getLogRecords(session, fromDate, toDate, event, pidFilter, start, count); + + OutputStream out = response.getOutputStream(); + response.setStatus(200); + response.setContentType("text/xml"); + + TypeMarshaller.marshalTypeToOutputStream(log, out); + IOUtils.closeQuietly(out); + } + + + /** + * Implements REST version of DataONE CRUD API --> get + * + * @param pid ID of data object to be read + * @throws NotImplemented + * @throws InvalidRequest + * @throws NotFound + * @throws NotAuthorized + * @throws ServiceFailure + * @throws InvalidToken + * @throws IOException + * @throws MarshallingException + */ + protected void getObject(String pid) throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, InvalidRequest, NotImplemented, IOException, MarshallingException { + OutputStream out = null; + + if (pid != null) { //get a specific document + long start = System.currentTimeMillis(); + Identifier id = new Identifier(); + id.setValue(pid); + + SystemMetadata sm = MNodeService.getInstance(request).getSystemMetadata(session, id); + + // set the headers for the content + String mimeType = null; + String charset = null; + ObjectFormat objectFormat = null; + + try { + objectFormat = ObjectFormatCache.getInstance().getFormat(sm.getFormatId()); + } catch (BaseException be) { + logMetacat.warn("Could not lookup ObjectFormat for: " + sm.getFormatId(), be); + } + // do we have mediaType/encoding in SM? + MediaType mediaType = sm.getMediaType(); + if (mediaType == null && objectFormat != null) { + try { + mediaType = objectFormat.getMediaType(); + } catch (Exception e) { + logMetacat.warn("Could not lookup MediaType for: " + sm.getFormatId(), e); + } + } + if (mediaType != null) { + mimeType = mediaType.getName(); + if (mediaType.getPropertyList() != null) { + Iterator iter = mediaType.getPropertyList().iterator(); + while (iter.hasNext()) { + MediaTypeProperty mtp = iter.next(); + if (mtp.getName().equalsIgnoreCase("charset")) { + charset = mtp.getValue(); + mimeType += "; charset=" + charset; + break; + } + } + } + } + // check object format + + // use the fallback from v1 impl + if (mimeType == null) { + mimeType = ObjectFormatInfo.instance().getMimeType(sm.getFormatId().getValue()); + + // still null? + if (mimeType == null) { + mimeType = "application/octet-stream"; + } + } + + // check for filename in SM first + String filename = sm.getFileName(); + // then fallback to using id and extension + if (filename == null) { + String extension = null; + if (objectFormat != null) { + extension = objectFormat.getExtension(); + } + if (extension == null) { + extension = ObjectFormatInfo.instance().getExtension(sm.getFormatId().getValue()); + } + filename = id.getValue(); + if (extension != null && filename != null && !filename.endsWith(extension)) { + filename = id.getValue() + "." + extension; + } + } + response.setContentType(mimeType); + response.setHeader("Content-Disposition", "inline; filename=\"" + filename + "\""); + InputStream data = null; + try { + data = MNodeService.getInstance(request).get(session, id); + out = response.getOutputStream(); + response.setStatus(200); + IOUtils.copyLarge(data, out); + IOUtils.closeQuietly(out); + } finally { + if (data != null) { + IOUtils.closeQuietly(data); + } + } + long end = System.currentTimeMillis(); + logMetacat.info(Settings.PERFORMANCELOG + pid + Settings.PERFORMANCELOG_GET_METHOD + " Total get method" + Settings.PERFORMANCELOG_DURATION + (end - start) / 1000); + } else { //call listObjects with specified params + Date startTime = null; + Date endTime = null; + ObjectFormatIdentifier formatId = null; + Identifier identifier = null; + boolean replicaStatus = true; + int start = 0; + //TODO: make the max count into a const + int count = 1000; + Enumeration paramlist = request.getParameterNames(); + while (paramlist.hasMoreElements()) { //parse the params and make the crud call + String name = (String) paramlist.nextElement(); + String[] value = (String[]) request.getParameterValues(name); + + if (name.equals("fromDate") && value != null) { + try { + //startTime = dateFormat.parse(value[0]); + startTime = DateTimeMarshaller.deserializeDateToUTC(value[0]); + //startTime = parseDateAndConvertToGMT(value[0]); + } catch (Exception e) { //if we can't parse it, just don't use the fromDate param + logMetacat.warn("Could not parse fromDate: " + value[0], e); + throw new InvalidRequest("1540", "Could not parse fromDate: " + value[0] + " since " + e.getMessage()); + //startTime = null; + } + } else if (name.equals("toDate") && value != null) { + try { + endTime = DateTimeMarshaller.deserializeDateToUTC(value[0]); + } catch (Exception e) { //if we can't parse it, just don't use the toDate param + logMetacat.warn("Could not parse toDate: " + value[0], e); + throw new InvalidRequest("1540", "Could not parse toDate: " + value[0] + " since " + e.getMessage()); + //endTime = null; + } + } else if (name.equals("formatId") && value != null) { + formatId = new ObjectFormatIdentifier(); + formatId.setValue(value[0]); + } else if (name.equals("identifier") && value != null) { + identifier = new Identifier(); + identifier.setValue(value[0]); + } else if (name.equals("replicaStatus") && value != null) { + if (value != null && + value.length > 0 && + (value[0].equalsIgnoreCase("false") || value[0].equalsIgnoreCase("no"))) { + replicaStatus = false; + } + } else if (name.equals("start") && value != null) { + start = new Integer(value[0]).intValue(); + } else if (name.equals("count") && value != null) { + count = new Integer(value[0]).intValue(); + } + } + //make the crud call + logMetacat.debug("session: " + session + " startTime: " + startTime + + " endTime: " + endTime + " formatId: " + + formatId + " replicaStatus: " + replicaStatus + + " start: " + start + " count: " + count); + + ObjectList ol = + MNodeService.getInstance(request).listObjects(session, startTime, endTime, + formatId, identifier, replicaStatus, start, count); + + out = response.getOutputStream(); + response.setStatus(200); + response.setContentType("text/xml"); + // Serialize and write it to the output stream + TypeMarshaller.marshalTypeToOutputStream(ol, out); + IOUtils.closeQuietly(out); + } + + } + + + /** + * Retrieve data package as Bagit zip + * + * @param format + * @param pid The pid of the resource map defining the pacakage + * @throws NotImplemented + * @throws NotFound + * @throws NotAuthorized + * @throws ServiceFailure + * @throws InvalidToken + * @throws IOException + * @throws InvalidRequest + */ + protected void getPackage(String format, String pid) throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented, IOException, InvalidRequest { + + long start = System.currentTimeMillis(); + Identifier id = new Identifier(); + id.setValue(pid); + ObjectFormatIdentifier formatId = null; + if (format != null) { + formatId = new ObjectFormatIdentifier(); + formatId.setValue(format); + } + try { + SpeedBagIt speedBag = MNodeService.getInstance(request).getPackage(session, formatId, id); + //Use the pid as the file name prefix, replacing all non-word characters + String filename = pid.replaceAll("\\W", "_") + ".zip"; + response.setHeader("Content-Disposition", "inline; filename=\"" + filename + "\""); + response.setContentType("application/zip"); + response.setStatus(200); + OutputStream out = response.getOutputStream(); + + // write it to the output stream + IOUtils.copyLarge(is, out); + IOUtils.closeQuietly(out); + long end = System.currentTimeMillis(); + logMetacat.info(Settings.PERFORMANCELOG + pid + Settings.PERFORMANCELOG_GET_PACKAGE_METHOD + " Total getPackage method" + Settings.PERFORMANCELOG_DURATION + (end - start) / 1000); + } finally { + IOUtils.closeQuietly(is); + } + } - long start = System.currentTimeMillis(); - Identifier id = new Identifier(); - id.setValue(pid); - ObjectFormatIdentifier formatId = null; - if (format != null) { - formatId = new ObjectFormatIdentifier(); - formatId.setValue(format); - } - try{ - SpeedBagIt speedBag = MNodeService.getInstance(request).getPackage(session, formatId , id); - - //Use the pid as the file name prefix, replacing all non-word characters - String filename = pid.replaceAll("\\W", "_") + ".zip"; - response.setHeader("Content-Disposition", "inline; filename=\"" + filename + "\""); - response.setContentType("application/zip"); - response.setStatus(200); - OutputStream out = response.getOutputStream(); - - // write it to the output stream - IOUtils.copyLarge(is, out); - IOUtils.closeQuietly(out); - long end = System.currentTimeMillis(); - logMetacat.info(Settings.PERFORMANCELOG + pid + Settings.PERFORMANCELOG_GET_PACKAGE_METHOD + " Total getPackage method" + Settings.PERFORMANCELOG_DURATION + (end - start) / 1000); - } finally { - IOUtils.closeQuietly(is); - } - } -======= - response.setHeader("Content-Disposition", "inline; filename=\"" + filename+"\""); - response.setContentType("application/zip"); - response.setStatus(200); - ZipOutputStream out = response.getOutputStream(); - - // write it to the output stream - speedBag.stream(out); - long end = System.currentTimeMillis(); - logMetacat.info(Settings.PERFORMANCELOG + pid + Settings.PERFORMANCELOG_GET_PACKAGE_METHOD + " Total getPackage method" + Settings.PERFORMANCELOG_DURATION + (end-start)/1000); - } ->>>>>>> Replace InputStream references with SpeedBagIt and use ZipOutputStream for client request - protected void publish(String pid) throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented, IOException, MarshallingException, InvalidRequest, IdentifierNotUnique, From 1944dc7125b18d823465641b65e2ff99df4b4c71 Mon Sep 17 00:00:00 2001 From: Thomas Thelen Date: Mon, 1 Mar 2021 09:43:32 -0800 Subject: [PATCH 10/45] Revert to using InputStream in getPackage --- .../nceas/metacat/dataone/MNodeService.java | 9 ++++--- .../restservice/v1/MNResourceHandler.java | 10 +++---- .../restservice/v2/MNResourceHandler.java | 27 ++++++++++++++++++- 3 files changed, 36 insertions(+), 10 deletions(-) diff --git a/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java b/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java index 0dad86a15..3c6d23f80 100644 --- a/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java +++ b/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java @@ -2553,7 +2553,7 @@ private List lookupOreFor(Session session, Identifier guid) { * @throws NotFound */ @Override - public SpeedBagIt getPackage(Session session, ObjectFormatIdentifier formatId, + public InputStream getPackage(Session session, ObjectFormatIdentifier formatId, Identifier pid) throws InvalidToken, ServiceFailure, NotAuthorized, InvalidRequest, NotImplemented, NotFound { if(formatId == null) { @@ -2781,7 +2781,7 @@ public SpeedBagIt getPackage(Session session, ObjectFormatIdentifier formatId, // Add the stream of the file to the bag object & write to the pid mapping file InputStream entryInputStream = this.get(session, entryPid); speedBag.addFile(entryInputStream, Paths.get("data/", fileName).toString(), false); - pidMapping.append(entryPid.getValue() + "\t" + "data" + fileName + "\n"); + pidMapping.append(entryPid.getValue() + "\t" + "data/" + fileName + "\n"); } // Get a stream to the pid mapping file and add it as a tag file, in the bag root @@ -2815,8 +2815,9 @@ public SpeedBagIt getPackage(Session session, ObjectFormatIdentifier formatId, throw sf; } - // Now that all of the files have been added to speedBag, return the InputStream - return speedBag; + // Now that all of the files have been added to speedBag, + + return speedBag.stream(); } /** diff --git a/src/edu/ucsb/nceas/metacat/restservice/v1/MNResourceHandler.java b/src/edu/ucsb/nceas/metacat/restservice/v1/MNResourceHandler.java index 2f1fad6f4..74409483a 100644 --- a/src/edu/ucsb/nceas/metacat/restservice/v1/MNResourceHandler.java +++ b/src/edu/ucsb/nceas/metacat/restservice/v1/MNResourceHandler.java @@ -1236,7 +1236,7 @@ protected void getPackage(String pid) throws InvalidToken, ServiceFailure, NotAu Identifier id = new Identifier(); id.setValue(pid); - SpeedBagIt speedBag = MNodeService.getInstance(request).getPackage(session, null, id); + InputStream is = MNodeService.getInstance(request).getPackage(session, null, id); //Use the pid as the file name prefix, replacing all non-word characters String filename = pid.replaceAll("\\W", "_"); @@ -1244,10 +1244,10 @@ protected void getPackage(String pid) throws InvalidToken, ServiceFailure, NotAu response.setHeader("Content-Disposition", "inline; filename=\"" + filename+"\""); response.setContentType("application/zip"); response.setStatus(200); - ZipOutputStream out = new ZipOutputStream(response.getOutputStream()); - - // write it to the output stream - speedBag.stream(out); + OutputStream out = response.getOutputStream(); + + // write it to the output stream + IOUtils.copyLarge(is, out); } protected void publish(String pid) throws InvalidToken, ServiceFailure, diff --git a/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java b/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java index ed026db66..10bac44a2 100644 --- a/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java +++ b/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java @@ -1294,11 +1294,36 @@ private void getLog() throws InvalidToken, ServiceFailure, NotAuthorized, Invali logMetacat.warn("Could not parse count: " + e.getMessage()); } +<<<<<<< HEAD try { pidFilter = params.get("idFilter")[0]; } catch (Exception e) { logMetacat.warn("Could not parse pidFilter: " + e.getMessage()); } +======= + /** + * Retrieve data package as Bagit zip + * @param format + * @param pid The pid of the resource map defining the pacakage + * @throws NotImplemented + * @throws NotFound + * @throws NotAuthorized + * @throws ServiceFailure + * @throws InvalidToken + * @throws IOException + * @throws InvalidRequest + */ + protected void getPackage(String format, String pid) throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented, IOException, InvalidRequest { + long start = System.currentTimeMillis(); + Identifier id = new Identifier(); + id.setValue(pid); + ObjectFormatIdentifier formatId = null; + if (format != null) { + formatId = new ObjectFormatIdentifier(); + formatId.setValue(format); + } + InputStream speedBag = MNodeService.getInstance(request).getPackage(session, formatId , id); +>>>>>>> Revert to using InputStream in getPackage logMetacat.debug("calling getLogRecords"); Log log = MNodeService.getInstance(request).getLogRecords(session, fromDate, toDate, event, pidFilter, start, count); @@ -1524,7 +1549,7 @@ protected void getPackage(String format, String pid) throws InvalidToken, Servic IOUtils.closeQuietly(is); } } - + protected void publish(String pid) throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented, IOException, MarshallingException, InvalidRequest, IdentifierNotUnique, From db44560b85258b5c35659fa1dec84301e1c2a2d4 Mon Sep 17 00:00:00 2001 From: Jing Tao Date: Thu, 25 Feb 2021 11:07:27 -0800 Subject: [PATCH 11/45] Add the code to remove the lib/maven directory in the init method. --- build.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/build.xml b/build.xml index 6bec47a87..e5d71dec2 100755 --- a/build.xml +++ b/build.xml @@ -254,6 +254,7 @@ + From 6d5c2dddaaf84680d83026b506e2226304e9035b Mon Sep 17 00:00:00 2001 From: Thomas Thelen Date: Mon, 1 Mar 2021 09:47:37 -0800 Subject: [PATCH 12/45] Replace SpeedBagIt with InputStream --- src/edu/ucsb/nceas/metacat/dataone/v1/MNodeService.java | 2 +- test/edu/ucsb/nceas/metacat/dataone/MNodeServiceTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/edu/ucsb/nceas/metacat/dataone/v1/MNodeService.java b/src/edu/ucsb/nceas/metacat/dataone/v1/MNodeService.java index 03bce3300..bf41c6fcf 100644 --- a/src/edu/ucsb/nceas/metacat/dataone/v1/MNodeService.java +++ b/src/edu/ucsb/nceas/metacat/dataone/v1/MNodeService.java @@ -573,7 +573,7 @@ public InputStream view(Session session, String format, Identifier pid) return impl.view(session, format, pid); } - public SpeedBagIt getPackage(Session session, ObjectFormatIdentifier formatId, + public InputStream getPackage(Session session, ObjectFormatIdentifier formatId, Identifier pid) throws InvalidToken, ServiceFailure, NotAuthorized, InvalidRequest, NotImplemented, NotFound { String serviceFailure = "2871"; diff --git a/test/edu/ucsb/nceas/metacat/dataone/MNodeServiceTest.java b/test/edu/ucsb/nceas/metacat/dataone/MNodeServiceTest.java index b8d515f18..fcc201c9c 100644 --- a/test/edu/ucsb/nceas/metacat/dataone/MNodeServiceTest.java +++ b/test/edu/ucsb/nceas/metacat/dataone/MNodeServiceTest.java @@ -1757,7 +1757,7 @@ public void testGetPackage() { Identifier pid = MNodeService.getInstance(request).create(session, guid, object, sysmeta); ObjectFormatIdentifier format = new ObjectFormatIdentifier(); format.setValue("application/bagit-097"); - SpeedBagIt bagStream = MNodeService.getInstance(request).getPackage(session, format, pid); + InputStream bagStream = MNodeService.getInstance(request).getPackage(session, format, pid); /*File bagFile = File.createTempFile("bagit.", ".zip"); IOUtils.copy(bagStream, new FileOutputStream(bagFile)); BagFactory bagFactory = new BagFactory(); From cc5839a548ef33a3ee3d827a849efdf9955f9086 Mon Sep 17 00:00:00 2001 From: Thomas Thelen Date: Mon, 1 Mar 2021 10:34:49 -0800 Subject: [PATCH 13/45] Improve exception handling --- .../nceas/metacat/dataone/MNodeService.java | 36 +++++++++++++++---- .../restservice/v2/MNResourceHandler.java | 5 +++ 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java b/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java index 3c6d23f80..8cf8e0dec 100644 --- a/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java +++ b/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java @@ -35,6 +35,7 @@ import java.io.StringReader; import java.io.UnsupportedEncodingException; import java.io.Writer; +import java.lang.NullPointerException; import java.math.BigInteger; import java.net.URISyntaxException; import java.nio.charset.Charset; @@ -2567,15 +2568,17 @@ public InputStream getPackage(Session session, ObjectFormatIdentifier formatId, if(sid != null) { pid = sid; } - // Create a bag that is version 0.97 and has tag files that contain MD5 checksums - SpeedBagIt speedBag = new SpeedBagIt(0.97, "MD5"); // The pids of all of the objects in the package List packagePids = new ArrayList(); // catch non-D1 service errors and throw as ServiceFailures + SpeedBagIt speedBag = null; try { - //Create a map of dataone ids and file names + // Create a bag that is version 0.97 and has tag files that contain MD5 checksums + speedBag = new SpeedBagIt(0.97, "MD5"); + + //Create a map of dataone ids and file names Map fileNames = new HashMap(); // track the pid-to-file mapping @@ -2813,11 +2816,30 @@ public InputStream getPackage(Session session, ObjectFormatIdentifier formatId, "for the package is valid. " + e.getMessage()); sf.initCause(e); throw sf; - } - - // Now that all of the files have been added to speedBag, + } catch (NoSuchAlgorithmException e) { + e.printStackTrace(); + ServiceFailure sf = new ServiceFailure("1030", "There was an " + + "error while adding a file to the archive. Please ensure that the " + + "checksumming algorithm is supported." + e.getMessage()); + sf.initCause(e); + throw sf; + } - return speedBag.stream(); + try { + return speedBag.stream(); + } catch (NullPointerException | IOException e) { + e.printStackTrace(); + ServiceFailure sf = new ServiceFailure("1030", "There was an " + + "error while streaming the downloaded data package. " + e.getMessage()); + sf.initCause(e); + throw sf; + } catch (NoSuchAlgorithmException e) { + e.printStackTrace(); + ServiceFailure sf = new ServiceFailure("1030", "While creating the package " + + "download, an unsupported checksumming algorithm was encountered. " + e.getMessage()); + sf.initCause(e); + throw sf; + } } /** diff --git a/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java b/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java index 10bac44a2..b08d98b2d 100644 --- a/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java +++ b/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java @@ -1332,9 +1332,14 @@ protected void getPackage(String format, String pid) throws InvalidToken, Servic response.setStatus(200); response.setContentType("text/xml"); +<<<<<<< HEAD TypeMarshaller.marshalTypeToOutputStream(log, out); IOUtils.closeQuietly(out); } +======= + IOUtils.copyLarge(speedBag, response.getOutputStream()); + // write it to the output stream +>>>>>>> Improve exception handling /** From a3342d6a89738a747ec7704cc264cf119fce5cfd Mon Sep 17 00:00:00 2001 From: Thomas Thelen Date: Thu, 15 Apr 2021 12:55:43 -0700 Subject: [PATCH 14/45] Add zip extension to downloads & fix bug where two science metadata documents were being written --- .../nceas/metacat/dataone/MNodeService.java | 48 +++---------------- .../restservice/v2/MNResourceHandler.java | 9 +++- 2 files changed, 14 insertions(+), 43 deletions(-) diff --git a/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java b/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java index 8cf8e0dec..bbd403d1f 100644 --- a/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java +++ b/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java @@ -2626,7 +2626,6 @@ public InputStream getPackage(Session session, ObjectFormatIdentifier formatId, params.put("docid", new String[] {localId}); params.put("pid", new String[] {pid.getValue()}); params.put("displaymodule", new String[] {"printall"}); - transformer.transformXMLDocument( documentContent , sourceType, @@ -2636,7 +2635,6 @@ public InputStream getPackage(Session session, ObjectFormatIdentifier formatId, params, null //sessionid ); - // finally, get the HTML back ContentTypeByteArrayInputStream resultInputStream = new ContentTypeByteArrayInputStream(baos.toByteArray()); @@ -2687,7 +2685,7 @@ public InputStream getPackage(Session session, ObjectFormatIdentifier formatId, pidMapping.append(metadataID.getValue() + " (pdf)" + "\t" + "data/" + pdfFile.getName() + "\n"); } catch (Exception e) { - logMetacat.warn("There was an error generating the PDF file during a package export. " + + logMetacat.error("There was an error generating the PDF file during a package export. " + "Ensure that the package metadata is valid and supported.", e); } } @@ -2700,42 +2698,12 @@ public InputStream getPackage(Session session, ObjectFormatIdentifier formatId, InputStream emlStream = this.get(session, metadataID); parser.parse(emlStream); DataPackage dataPackage = parser.getDataPackage(); - - //Get all the entities in this package and loop through each to extract its ID and file name - Entity[] entities = dataPackage.getEntityList(); - for(Entity entity: entities){ - try{ - //Get the file name from the metadata - String fileNameFromMetadata = entity.getName(); - - //Get the ecogrid URL from the metadata - String ecogridIdentifier = entity.getEntityIdentifier(); - //Parse the ecogrid URL to get the local id - String idFromMetadata = DocumentUtil.getAccessionNumberFromEcogridIdentifier(ecogridIdentifier); - - //Get the docid and rev pair - String docid = DocumentUtil.getDocIdFromString(idFromMetadata); - String rev = DocumentUtil.getRevisionStringFromString(idFromMetadata); - - //Get the GUID - String guid = IdentifierManager.getInstance().getGUID(docid, Integer.valueOf(rev)); - Identifier dataIdentifier = new Identifier(); - dataIdentifier.setValue(guid); - - //Add the GUID to our GUID & file name map - fileNames.put(dataIdentifier, fileNameFromMetadata); - } - catch(Exception e){ - //Prevent just one entity error - e.printStackTrace(); - logMetacat.debug(e.getMessage(), e); - } - } + } } catch(Exception e){ //Catch errors that would prevent package download - logMetacat.debug(e.toString()); + logMetacat.error(e.toString()); } } packagePids.addAll(entries.keySet()); @@ -2756,21 +2724,17 @@ public InputStream getPackage(Session session, ObjectFormatIdentifier formatId, * The next step is looping over each object in the package, determining its filename, * getting an InputStream to it, and adding it to the bag. */ - for (Identifier entryPid: packagePids) { + Set packagePidsUnique = new HashSet<>(packagePids); + for (Identifier entryPid: packagePidsUnique) { //Get the system metadata for each item SystemMetadata entrySysMeta = this.getSystemMetadata(session, entryPid); String objectFormatType = ObjectFormatCache.getInstance().getFormat(entrySysMeta.getFormatId()).getFormatType(); String fileName = null; - + //TODO: Be more specific of what characters to replace. Make sure periods arent replaced for the filename from metadata //Our default file name is just the ID + format type (e.g. walker.1.1-DATA) fileName = entryPid.getValue().replaceAll("[^a-zA-Z0-9\\-\\.]", "_") + "-" + objectFormatType; - - if (fileNames.containsKey(entryPid)){ - //Let's use the file name and extension from the metadata is we have it - fileName = entryPid.getValue().replaceAll("[^a-zA-Z0-9\\-\\.]", "_") + "-" + fileNames.get(entryPid).replaceAll("[^a-zA-Z0-9\\-\\.]", "_"); - } // ensure there is a file extension for the object String extension = ObjectFormatInfo.instance().getExtension(entrySysMeta.getFormatId().getValue()); diff --git a/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java b/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java index b08d98b2d..9c88534a1 100644 --- a/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java +++ b/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java @@ -1325,6 +1325,7 @@ protected void getPackage(String format, String pid) throws InvalidToken, Servic InputStream speedBag = MNodeService.getInstance(request).getPackage(session, formatId , id); >>>>>>> Revert to using InputStream in getPackage +<<<<<<< HEAD logMetacat.debug("calling getLogRecords"); Log log = MNodeService.getInstance(request).getLogRecords(session, fromDate, toDate, event, pidFilter, start, count); @@ -1337,6 +1338,13 @@ protected void getPackage(String format, String pid) throws InvalidToken, Servic IOUtils.closeQuietly(out); } ======= +======= + //Use the pid as the file name prefix, replacing all non-word characters + String filename = pid.replaceAll("\\W", "_") + ".zip"; + response.setHeader("Content-Disposition", "inline; filename=\"" + filename+"\""); + response.setContentType("application/zip"); + response.setStatus(200); +>>>>>>> Add zip extension to downloads & fix bug where two science metadata documents were being written IOUtils.copyLarge(speedBag, response.getOutputStream()); // write it to the output stream >>>>>>> Improve exception handling @@ -1547,7 +1555,6 @@ protected void getPackage(String format, String pid) throws InvalidToken, Servic // write it to the output stream IOUtils.copyLarge(is, out); - IOUtils.closeQuietly(out); long end = System.currentTimeMillis(); logMetacat.info(Settings.PERFORMANCELOG + pid + Settings.PERFORMANCELOG_GET_PACKAGE_METHOD + " Total getPackage method" + Settings.PERFORMANCELOG_DURATION + (end - start) / 1000); } finally { From b23db9742f66966f01b7f0e9770c8bf8051875e3 Mon Sep 17 00:00:00 2001 From: Thomas Thelen Date: Thu, 15 Apr 2021 15:19:03 -0700 Subject: [PATCH 15/45] Fix rebase --- .../restservice/v2/MNResourceHandler.java | 45 ++----------------- 1 file changed, 3 insertions(+), 42 deletions(-) diff --git a/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java b/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java index 9c88534a1..b53518b69 100644 --- a/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java +++ b/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java @@ -1293,39 +1293,12 @@ private void getLog() throws InvalidToken, ServiceFailure, NotAuthorized, Invali } catch (Exception e) { logMetacat.warn("Could not parse count: " + e.getMessage()); } - -<<<<<<< HEAD try { pidFilter = params.get("idFilter")[0]; } catch (Exception e) { logMetacat.warn("Could not parse pidFilter: " + e.getMessage()); } -======= - /** - * Retrieve data package as Bagit zip - * @param format - * @param pid The pid of the resource map defining the pacakage - * @throws NotImplemented - * @throws NotFound - * @throws NotAuthorized - * @throws ServiceFailure - * @throws InvalidToken - * @throws IOException - * @throws InvalidRequest - */ - protected void getPackage(String format, String pid) throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented, IOException, InvalidRequest { - long start = System.currentTimeMillis(); - Identifier id = new Identifier(); - id.setValue(pid); - ObjectFormatIdentifier formatId = null; - if (format != null) { - formatId = new ObjectFormatIdentifier(); - formatId.setValue(format); - } - InputStream speedBag = MNodeService.getInstance(request).getPackage(session, formatId , id); ->>>>>>> Revert to using InputStream in getPackage -<<<<<<< HEAD logMetacat.debug("calling getLogRecords"); Log log = MNodeService.getInstance(request).getLogRecords(session, fromDate, toDate, event, pidFilter, start, count); @@ -1333,22 +1306,9 @@ protected void getPackage(String format, String pid) throws InvalidToken, Servic response.setStatus(200); response.setContentType("text/xml"); -<<<<<<< HEAD TypeMarshaller.marshalTypeToOutputStream(log, out); IOUtils.closeQuietly(out); } -======= -======= - //Use the pid as the file name prefix, replacing all non-word characters - String filename = pid.replaceAll("\\W", "_") + ".zip"; - response.setHeader("Content-Disposition", "inline; filename=\"" + filename+"\""); - response.setContentType("application/zip"); - response.setStatus(200); ->>>>>>> Add zip extension to downloads & fix bug where two science metadata documents were being written - IOUtils.copyLarge(speedBag, response.getOutputStream()); - // write it to the output stream ->>>>>>> Improve exception handling - /** * Implements REST version of DataONE CRUD API --> get @@ -1544,8 +1504,9 @@ protected void getPackage(String format, String pid) throws InvalidToken, Servic formatId = new ObjectFormatIdentifier(); formatId.setValue(format); } + InputStream is = null; try { - SpeedBagIt speedBag = MNodeService.getInstance(request).getPackage(session, formatId, id); + is = MNodeService.getInstance(request).getPackage(session, formatId , id); //Use the pid as the file name prefix, replacing all non-word characters String filename = pid.replaceAll("\\W", "_") + ".zip"; response.setHeader("Content-Disposition", "inline; filename=\"" + filename + "\""); @@ -1561,7 +1522,7 @@ protected void getPackage(String format, String pid) throws InvalidToken, Servic IOUtils.closeQuietly(is); } } - + protected void publish(String pid) throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented, IOException, MarshallingException, InvalidRequest, IdentifierNotUnique, From bfc7998139afffbe3dc34f4f1a4ca1bb9452503e Mon Sep 17 00:00:00 2001 From: Matt Jones Date: Thu, 8 Jul 2021 00:09:45 -0800 Subject: [PATCH 16/45] Proposed refactor to support configuring multiple DOI shoulders. The first shoulder is considered primary, and should use the property key `guid.ezid.shoulder.1`, and should be a minter capable shoulder. Subsequent shoulders should increment the suffix (e.g., `guid.ezid.sholder.2`, and can be for both minters and creation of IDs. Both cases will get DOI updates sent to EZID. See issue #1514. --- .../nceas/metacat/dataone/DOIService.java | 142 +++++++++++------- 1 file changed, 84 insertions(+), 58 deletions(-) diff --git a/src/edu/ucsb/nceas/metacat/dataone/DOIService.java b/src/edu/ucsb/nceas/metacat/dataone/DOIService.java index 377df4550..d6c3c203d 100644 --- a/src/edu/ucsb/nceas/metacat/dataone/DOIService.java +++ b/src/edu/ucsb/nceas/metacat/dataone/DOIService.java @@ -23,6 +23,7 @@ package edu.ucsb.nceas.metacat.dataone; import java.io.InputStream; +import java.lang.Integer; import java.text.SimpleDateFormat; import java.util.Arrays; import java.util.Calendar; @@ -74,55 +75,55 @@ import edu.ucsb.nceas.utilities.StringUtil; /** - * + * * Singleton for interacting with the EZID DOI library. - * Allows DOI minting/initial registration, creating and updating + * Allows DOI minting/initial registration, creating and updating * existing DOI registrations. - * + * * @author leinfelder */ public class DOIService { - + public static final String DATACITE = "datacite"; private Log logMetacat = LogFactory.getLog(DOIService.class); private boolean doiEnabled = false; - - private String shoulder = null; - + + private HashMap shoulderMap = null; + private String ezidUsername = null; - + private String ezidPassword = null; - + private EZIDClient ezid = null; - + private Date lastLogin = null; - + private long loginPeriod = 1 * 24 * 60 * 60 * 1000; private static DOIService instance = null; - + private Vector dataCiteFactories = new Vector(); - + public static DOIService getInstance() { if (instance == null) { instance = new DOIService(); } return instance; } - + /** * Constructor, private for singleton access */ private DOIService() { - + // for DOIs String ezidServiceBaseUrl = null; - + shoulderMap = new HashMap(); + try { doiEnabled = new Boolean(PropertyService.getProperty("guid.ezid.enabled")).booleanValue(); - shoulder = PropertyService.getProperty("guid.ezid.doishoulder.1"); ezidServiceBaseUrl = PropertyService.getProperty("guid.ezid.baseurl"); ezidUsername = PropertyService.getProperty("guid.ezid.username"); ezidPassword = PropertyService.getProperty("guid.ezid.password"); @@ -130,10 +131,28 @@ private DOIService() { logMetacat.warn("DOI support is not configured at this node.", e); return; } + + boolean moreShoulders = true; + int i=0; + while (moreShoulders) { + i++; + try { + String shoulder = PropertyService.getProperty("guid.ezid.doishoulder."+i); + shoulderMap.put(new Integer(i), shoulder); + } catch (PropertyNotFoundException e) { + moreShoulders = false; + } + } + + if (shoulderMap.size() < 1) { + logMetacat.warn("DOI support is not configured at this node because no shoulders are configured."); + return; + } + ezid = new EZIDClient(ezidServiceBaseUrl); initDataCiteFactories(); } - + /* * Initialize the datacite factory by reading the property guid.ezid.datacite.factories from the metacat.properties file. */ @@ -158,12 +177,12 @@ private void initDataCiteFactories() { logMetacat.debug("DOIService.initDataCiteFactories - the DataCiteFactory " + factoryClass + " was initialized."); } catch (Exception e) { logMetacat.warn("DOIService.initDataCiteFactories - can't initialize the class " + factoryClass + " since "+e.getMessage()); - } + } } } } } - + /** * Make sure we have a current login before making any calls * @throws EZIDException @@ -172,21 +191,21 @@ private void refreshLogin() throws EZIDException { Date now = Calendar.getInstance().getTime(); if (lastLogin == null || now.getTime() - lastLogin.getTime() > loginPeriod) { ezid.login(ezidUsername, ezidPassword); - lastLogin = now; + lastLogin = now; } } - + /** * submits DOI metadata information about the object to EZID * @param sysMeta * @return - * @throws EZIDException - * @throws ServiceFailure - * @throws NotImplemented - * @throws InterruptedException + * @throws EZIDException + * @throws ServiceFailure + * @throws NotImplemented + * @throws InterruptedException */ public boolean registerDOI(SystemMetadata sysMeta) throws InvalidRequest, EZIDException, NotImplemented, ServiceFailure, InterruptedException { - + // only continue if we have the feature turned on if (doiEnabled) { boolean identifierIsDOI = false; @@ -196,17 +215,19 @@ public boolean registerDOI(SystemMetadata sysMeta) throws InvalidRequest, EZIDEx if(sysMeta.getSeriesId() != null) { sid = sysMeta.getSeriesId().getValue(); } - - // determine if this DOI identifier is in our configured shoulder - if (shoulder != null && !shoulder.trim().equals("") && identifier != null && identifier.startsWith(shoulder)) { - identifierIsDOI = true; - } - // determine if this DOI sid is in our configured shoulder - if (shoulder != null && !shoulder.trim().equals("") && sid != null && sid.startsWith(shoulder)) { - sidIsDOI = true; + + // determine if this DOI identifier is in our configured list of shoulders + for (String shoulder : shoulderMap.values()) { + if (shoulder != null && !shoulder.trim().equals("") && identifier != null && identifier.startsWith(shoulder)) { + identifierIsDOI = true; + } + // determine if this DOI sid is in our configured shoulder + if (shoulder != null && !shoulder.trim().equals("") && sid != null && sid.startsWith(shoulder)) { + sidIsDOI = true; + } } - - // only continue if this DOI identifier or sid is in our configured shoulder + + // only continue if this DOI identifier or sid is in our configured shoulder list if(identifierIsDOI || sidIsDOI) { // finish the other part for the identifier if it is an DOI if(identifierIsDOI) { @@ -217,28 +238,28 @@ public boolean registerDOI(SystemMetadata sysMeta) throws InvalidRequest, EZIDEx registerDOI(sid, sysMeta); } } - + } - + return true; } - + /** * Register the metadata for the given identifier. The given identifier can be an SID. * @param identifier the given identifier will be registered with the metadata * @param title the title will be in the metadata - * @param sysMeta the system metadata associates with the given id + * @param sysMeta the system metadata associates with the given id * @param creators the creator will be in the metadata - * @throws ServiceFailure - * @throws NotImplemented - * @throws EZIDException - * @throws InterruptedException + * @throws ServiceFailure + * @throws NotImplemented + * @throws EZIDException + * @throws InterruptedException */ private void registerDOI(String identifier, SystemMetadata sysMeta) throws InvalidRequest, NotImplemented, ServiceFailure, EZIDException, InterruptedException { // enter metadata about this identifier HashMap metadata = new HashMap(); Node node = MNodeService.getInstance(null).getCapabilities(); - + // target (URL) String target = node.getBaseURL() + "/v1/object/" + identifier; String uriTemplate = null; @@ -258,7 +279,7 @@ private void registerDOI(String identifier, SystemMetadata sysMeta) throws Inval } catch (PropertyNotFoundException e) { logMetacat.warn("No target URI template found in the configuration for: " + uriTemplateKey); } - + // status and export fields for public/protected data String status = InternalProfileValues.UNAVAILABLE.toString(); String export = InternalProfileValues.NO.toString(); @@ -268,7 +289,7 @@ private void registerDOI(String identifier, SystemMetadata sysMeta) throws Inval status = InternalProfileValues.PUBLIC.toString(); export = InternalProfileValues.YES.toString(); } - + // set the datacite metadata fields String dataCiteXML = generateDataCiteXML(identifier, sysMeta); metadata.put(DATACITE, dataCiteXML); @@ -278,19 +299,19 @@ private void registerDOI(String identifier, SystemMetadata sysMeta) throws Inval // make sure we have a current login this.refreshLogin(); - + // set using the API ezid.createOrUpdate(identifier, metadata); } - + /** * Generate the datacite xml document for the given information. * This method will look at the registered datacite factories to find a proper one for the given meta data standard. - * If it can't find it, the default factory will be used. + * If it can't find it, the default factory will be used. * @param identifier * @param sysmeta * @return - * @throws ServiceFailure + * @throws ServiceFailure */ private String generateDataCiteXML(String identifier, SystemMetadata sysMeta) throws InvalidRequest, ServiceFailure { Identifier id = new Identifier(); @@ -308,17 +329,17 @@ private String generateDataCiteXML(String identifier, SystemMetadata sysMeta) th /** * Generate a DOI using the EZID service as configured * @return - * @throws EZIDException - * @throws InvalidRequest + * @throws EZIDException + * @throws InvalidRequest */ public Identifier generateDOI() throws EZIDException, InvalidRequest { - + // only continue if we have the feature turned on if (!doiEnabled) { throw new InvalidRequest("2193", "DOI scheme is not enabled at this node."); } - + // add only the minimal metadata required for this DOI HashMap metadata = new HashMap(); metadata.put(DataCiteProfile.TITLE.toString(), ErcMissingValueCode.UNKNOWN.toString()); @@ -331,11 +352,16 @@ public Identifier generateDOI() throws EZIDException, InvalidRequest { // make sure we have a current login this.refreshLogin(); + // Make sure we have a primary shoulder configured (which should enable mint operations) + if (!shoulderMap.containsKey(new Integer(1))) { + throw new InvalidRequest("2193", "DOI scheme is not enabled at this node because primary shoulder unconfigured."); + } + // call the EZID service - String doi = ezid.mintIdentifier(shoulder, metadata); + String doi = ezid.mintIdentifier(shoulderMap.get(new Integer(1)), metadata); Identifier identifier = new Identifier(); identifier.setValue(doi); - + return identifier; } From 08423c6ca731a2385e999b9cd71fcf7e26be8c9a Mon Sep 17 00:00:00 2001 From: Jing Tao Date: Mon, 12 Jul 2021 11:52:46 -0700 Subject: [PATCH 17/45] Use a constance as the index number of the primary shoulder. --- .../ucsb/nceas/metacat/dataone/DOIService.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/edu/ucsb/nceas/metacat/dataone/DOIService.java b/src/edu/ucsb/nceas/metacat/dataone/DOIService.java index d6c3c203d..65ae86078 100644 --- a/src/edu/ucsb/nceas/metacat/dataone/DOIService.java +++ b/src/edu/ucsb/nceas/metacat/dataone/DOIService.java @@ -103,6 +103,8 @@ public class DOIService { private long loginPeriod = 1 * 24 * 60 * 60 * 1000; private static DOIService instance = null; + + private static int PRIMARY_SHOULDER_INDEX = 1; private Vector dataCiteFactories = new Vector(); @@ -133,12 +135,16 @@ private DOIService() { } boolean moreShoulders = true; - int i=0; + int i = PRIMARY_SHOULDER_INDEX; while (moreShoulders) { - i++; try { - String shoulder = PropertyService.getProperty("guid.ezid.doishoulder."+i); - shoulderMap.put(new Integer(i), shoulder); + String shoulder = PropertyService.getProperty("guid.ezid.doishoulder." + i); + if (shoulder != null && !shoulder.trim().equals("")) { + logMetacat.debug("DOIService.constructor - add the shoulder " + shoulder + + " with the key " + i + " into the shoulder map. "); + shoulderMap.put(new Integer(i), shoulder); + } + i++; } catch (PropertyNotFoundException e) { moreShoulders = false; } @@ -353,12 +359,12 @@ public Identifier generateDOI() throws EZIDException, InvalidRequest { this.refreshLogin(); // Make sure we have a primary shoulder configured (which should enable mint operations) - if (!shoulderMap.containsKey(new Integer(1))) { + if (!shoulderMap.containsKey(new Integer(PRIMARY_SHOULDER_INDEX))) { throw new InvalidRequest("2193", "DOI scheme is not enabled at this node because primary shoulder unconfigured."); } // call the EZID service - String doi = ezid.mintIdentifier(shoulderMap.get(new Integer(1)), metadata); + String doi = ezid.mintIdentifier(shoulderMap.get(new Integer(PRIMARY_SHOULDER_INDEX)), metadata); Identifier identifier = new Identifier(); identifier.setValue(doi); From d70e70cb65c99327be5794fba20d05f5a1e6f070 Mon Sep 17 00:00:00 2001 From: Thomas Thelen Date: Wed, 21 Jul 2021 16:36:18 -0700 Subject: [PATCH 18/45] Convert tabs to spaces --- .../restservice/v2/MNResourceHandler.java | 2878 ++++++++--------- 1 file changed, 1439 insertions(+), 1439 deletions(-) diff --git a/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java b/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java index b53518b69..bacf3f973 100644 --- a/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java +++ b/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java @@ -147,673 +147,673 @@ */ public class MNResourceHandler extends D1ResourceHandler { - // MN-specific API Resources - protected static final String RESOURCE_MONITOR = "monitor"; - protected static final String RESOURCE_REPLICATE = "replicate"; - protected static final String RESOURCE_REPLICAS = "replica"; - protected static final String RESOURCE_NODE = "node"; - protected static final String RESOURCE_ERROR = "error"; - protected static final String RESOURCE_META_CHANGED = "dirtySystemMetadata"; - protected static final String RESOURCE_GENERATE_ID = "generate"; - protected static final String RESOURCE_PUBLISH = "publish"; - protected static final String RESOURCE_PACKAGE = "packages"; - protected static final String RESOURCE_TOKEN = "token"; - protected static final String RESOURCE_WHOAMI = "whoami"; - - - // shared executor - private static ExecutorService executor = null; - - static { - // use a shared executor service with nThreads == one less than available processors - int availableProcessors = Runtime.getRuntime().availableProcessors(); - int nThreads = availableProcessors * 1; - nThreads--; - nThreads = Math.max(1, nThreads); - executor = Executors.newFixedThreadPool(nThreads); - } - - /** - * Initializes new instance by setting servlet context,request and response - */ - public MNResourceHandler(ServletContext servletContext, - HttpServletRequest request, HttpServletResponse response) { - super(servletContext, request, response); - logMetacat = LogFactory.getLog(MNResourceHandler.class); - } - - @Override - protected boolean isD1Enabled() { - - boolean enabled = false; - try { - enabled = Boolean.parseBoolean(PropertyService.getProperty("dataone.mn.services.enabled")); - } catch (PropertyNotFoundException e) { - logMetacat.error("Could not check if DataONE is enabled: " + e.getMessage()); - } - - return enabled; - } - - /** - * This function is called from REST API servlet and handles each request to the servlet - * - * @param httpVerb (GET, POST, PUT or DELETE) - */ - @Override - public void handle(byte httpVerb) { - // prepare the handler - super.handle(httpVerb); - - try { - - // only service requests if we have D1 configured - if (!isD1Enabled()) { - ServiceFailure se = new ServiceFailure("0000", "DataONE services are not enabled on this node"); - serializeException(se, response.getOutputStream()); - return; - } - - // get the resource - String resource = request.getPathInfo(); - resource = resource.substring(resource.indexOf("/") + 1); - - // default to node info - if (resource.equals("")) { - resource = RESOURCE_NODE; - } - - // get the rest of the path info - String extra = null; - - logMetacat.info("MNResourceHandler.handle - V2 handling verb " + httpVerb + " request with resource '" + resource + "'"); - logMetacat.debug("resource: '" + resource + "'"); - boolean status = false; - - if (resource != null) { - - if (resource.startsWith(RESOURCE_NODE)) { - // node response - node(); - status = true; - } else if (resource.startsWith(RESOURCE_TOKEN)) { - logMetacat.debug("Using resource 'token'"); - // get - if (httpVerb == GET) { - // after the command - getToken(); - status = true; - } - - } else if (resource.startsWith(RESOURCE_WHOAMI)) { - logMetacat.debug("Using resource 'whoami'"); - // get - if (httpVerb == GET) { - // after the command - whoami(); - status = true; - } - - } else if (resource.startsWith(RESOURCE_IS_AUTHORIZED)) { - if (httpVerb == GET) { - // after the command - extra = parseTrailing(resource, RESOURCE_IS_AUTHORIZED); - extra = decode(extra); - // check the access rules - isAuthorized(extra); - status = true; - logMetacat.debug("done getting access"); - } - } else if (resource.startsWith(RESOURCE_META)) { - logMetacat.debug("Using resource 'meta'"); - // get - if (httpVerb == GET) { - logMetacat.debug("Using resource 'meta' for GET"); - // after the command - extra = parseTrailing(resource, RESOURCE_META); - extra = decode(extra); - getSystemMetadataObject(extra); - status = true; - } else if (httpVerb == PUT) { - logMetacat.debug("Using resource 'meta' for PUT"); - updateSystemMetadata(); - status = true; - } - - } else if (resource.startsWith(RESOURCE_OBJECTS)) { - logMetacat.debug("Using resource 'object'"); - // after the command - extra = parseTrailing(resource, RESOURCE_OBJECTS); - extra = decode(extra); - logMetacat.debug("objectId: " + extra); - logMetacat.debug("verb:" + httpVerb); - - if (httpVerb == GET) { - getObject(extra); - status = true; - } else if (httpVerb == POST) { - // part of the params, not the URL - putObject(null, FUNCTION_NAME_INSERT); - status = true; - } else if (httpVerb == PUT) { - putObject(extra, FUNCTION_NAME_UPDATE); - status = true; - } else if (httpVerb == DELETE) { - deleteObject(extra); - status = true; - } else if (httpVerb == HEAD) { - describeObject(extra); - status = true; - } - - } else if (resource.startsWith(RESOURCE_LOG)) { - logMetacat.debug("Using resource 'log'"); - // handle log events - if (httpVerb == GET) { - getLog(); - status = true; - } - } else if (resource.startsWith(Constants.RESOURCE_ARCHIVE)) { - logMetacat.debug("Using resource " + Constants.RESOURCE_ARCHIVE); - // handle archive events - if (httpVerb == PUT) { - extra = parseTrailing(resource, Constants.RESOURCE_ARCHIVE); - extra = decode(extra); - archive(extra); - status = true; - } - } else if (resource.startsWith(Constants.RESOURCE_CHECKSUM)) { - logMetacat.debug("Using resource 'checksum'"); - // handle checksum requests - if (httpVerb == GET) { - // after the command - extra = parseTrailing(resource, Constants.RESOURCE_CHECKSUM); - extra = decode(extra); - checksum(extra); - status = true; - } - } else if (resource.startsWith(RESOURCE_MONITOR)) { - // there are various parts to monitoring - if (httpVerb == GET) { - // after the command - extra = parseTrailing(resource, RESOURCE_MONITOR); - extra = decode(extra); - // ping - if (extra.toLowerCase().equals("ping")) { - logMetacat.debug("processing ping request"); - Date result = MNodeService.getInstance(request).ping(); - // TODO: send to output - status = true; - - } else if (extra.toLowerCase().equals("status")) { - logMetacat.debug("processing status request"); - getStatus(); - status = true; - } - - } - } else if (resource.startsWith(RESOURCE_REPLICATE)) { - if (httpVerb == POST) { - logMetacat.debug("processing replicate request"); - replicate(); - status = true; - } - } else if (resource.startsWith(RESOURCE_ERROR)) { - // sync error - if (httpVerb == POST) { - syncError(); - status = true; - } - } else if (resource.startsWith(RESOURCE_META_CHANGED)) { - // system metadata changed - if (httpVerb == POST) { - systemMetadataChanged(); - status = true; - } - } else if (resource.startsWith(RESOURCE_REPLICAS)) { - // get replica - if (httpVerb == GET) { - extra = parseTrailing(resource, RESOURCE_REPLICAS); - extra = decode(extra); - getReplica(extra); - status = true; - } - } else if (resource.startsWith(RESOURCE_QUERY)) { - logMetacat.debug("Using resource " + RESOURCE_QUERY); - // after the command - extra = parseTrailing(resource, RESOURCE_QUERY); - logMetacat.debug("query extra: " + extra); - - String engine = null; - String query = null; - - if (extra != null) { - // get the engine - int engineIndex = extra.length(); - if (extra.indexOf("/") > -1) { - engineIndex = extra.indexOf("/"); - } - engine = extra.substring(0, engineIndex); - engine = decode(engine); - logMetacat.debug("query engine: " + engine); - - // check the query string first - query = request.getQueryString(); - - // if null, look at the whole endpoint - if (query == null) { - // get the query if it is there - query = extra.substring(engineIndex, extra.length()); - if (query != null && query.length() == 0) { - query = null; - } else { - if (query.startsWith("/")) { - query = query.substring(1); - } - } - } - query = decode(query); - logMetacat.debug("query: " + query); - - } - logMetacat.debug("verb:" + httpVerb); - if (httpVerb == GET) { - doQuery(engine, query); - status = true; - } else if (httpVerb == POST) { - doPostQuery(engine); - status = true; - } - } else if (resource.startsWith(RESOURCE_GENERATE_ID)) { - // generate an id - if (httpVerb == POST) { - generateIdentifier(); - status = true; - } - } else if (resource.startsWith(RESOURCE_PUBLISH)) { - logMetacat.debug("Using resource: " + RESOURCE_PUBLISH); - // PUT - if (httpVerb == PUT) { - // after the command - extra = parseTrailing(resource, RESOURCE_PUBLISH); - extra = decode(extra); - publish(extra); - status = true; - } - } else if (resource.startsWith(RESOURCE_PACKAGE)) { - logMetacat.debug("Using resource: " + RESOURCE_PACKAGE); - // after the command - extra = parseTrailing(resource, RESOURCE_PACKAGE); - - String format = null; - String pid = null; - - if (extra != null) { - // get the format - int formatIndex = extra.length(); - if (extra.indexOf("/") > -1) { - formatIndex = extra.indexOf("/"); - } - format = extra.substring(0, formatIndex); - format = decode(format); - logMetacat.debug("package format: " + format); - - // get the pid if it is there - pid = extra.substring(formatIndex, extra.length()); - if (pid != null && pid.length() == 0) { - pid = null; - } else { - if (pid.startsWith("/")) { - pid = pid.substring(1); - } - } - pid = decode(pid); - logMetacat.debug("pid: " + pid); - - } - - // get - if (httpVerb == GET) { - - getPackage(format, pid); - status = true; - } - } else if (resource.startsWith(RESOURCE_VIEWS)) { - logMetacat.debug("Using resource " + RESOURCE_VIEWS); - // after the command - extra = parseTrailing(resource, RESOURCE_VIEWS); - logMetacat.debug("view extra: " + extra); - - String format = null; - String pid = null; - - if (extra != null) { - // get the format - int formatIndex = extra.length(); - if (extra.indexOf("/") > -1) { - formatIndex = extra.indexOf("/"); - } - format = extra.substring(0, formatIndex); - format = decode(format); - logMetacat.debug("view format: " + format); - - // get the pid if it is there - pid = extra.substring(formatIndex, extra.length()); - if (pid != null && pid.length() == 0) { - pid = null; - } else { - if (pid.startsWith("/")) { - pid = pid.substring(1); - } - } - pid = decode(pid); - logMetacat.debug("pid: " + pid); - - } - logMetacat.debug("verb:" + httpVerb); - if (httpVerb == GET) { - doViews(format, pid); - status = true; - } - } else { - throw new InvalidRequest("0000", "No resource matched for " + resource); - } - - if (!status) { - throw new ServiceFailure("0000", "Unknown error, status = " + status); - } - } else { - throw new InvalidRequest("0000", "No resource matched for " + resource); - } - } catch (BaseException be) { - // report Exceptions as clearly as possible - OutputStream out = null; - try { - out = response.getOutputStream(); - } catch (IOException e) { - logMetacat.error("Could not get output stream from response", e); - } - serializeException(be, out); - } catch (Exception e) { - // report Exceptions as clearly and generically as possible - logMetacat.error(e.getClass() + ": " + e.getMessage(), e); - OutputStream out = null; - try { - out = response.getOutputStream(); - } catch (IOException ioe) { - logMetacat.error("Could not get output stream from response", ioe); - } - ServiceFailure se = new ServiceFailure("0000", e.getMessage()); - serializeException(se, out); - } - } - - - private void doQuery(String engine, String query) { - - OutputStream out = null; - - try { - // NOTE: we set the session explicitly for the MNode instance since these methods do not provide a parameter - if (engine == null) { - // just looking for list of engines - MNodeService mnode = MNodeService.getInstance(request); - mnode.setSession(session); - QueryEngineList qel = mnode.listQueryEngines(session); - response.setContentType("text/xml"); - response.setStatus(200); - out = response.getOutputStream(); - TypeMarshaller.marshalTypeToOutputStream(qel, out); - IOUtils.closeQuietly(out); - return; - } else { - if (query != null) { - long start = System.currentTimeMillis(); - MNodeService mnode = MNodeService.getInstance(request); - mnode.setSession(session); - InputStream stream = mnode.query(session, engine, query); - - // set the content-type if we have it from the implementation - if (stream instanceof ContentTypeInputStream) { - //response.setContentType("application/octet-stream"); - //response.setContentType("text/xml"); - response.setContentType(((ContentTypeInputStream) stream).getContentType()); - } - response.setStatus(200); - out = response.getOutputStream(); - // write the results to the output stream - IOUtils.copyLarge(stream, out); - long end = System.currentTimeMillis(); - logMetacat.info(Settings.PERFORMANCELOG + Settings.PERFORMANCELOG_QUERY_METHOD + query + " Total query method" + Settings.PERFORMANCELOG_DURATION + (end - start) / 1000); - IOUtils.closeQuietly(out); - return; - } else { - MNodeService mnode = MNodeService.getInstance(request); - mnode.setSession(session); - QueryEngineDescription qed = mnode.getQueryEngineDescription(session, engine); - response.setContentType("text/xml"); - response.setStatus(200); - out = response.getOutputStream(); - TypeMarshaller.marshalTypeToOutputStream(qed, out); - IOUtils.closeQuietly(out); - return; - } - } - - - } catch (BaseException be) { - // report Exceptions as clearly as possible - try { - out = response.getOutputStream(); - } catch (IOException e) { - logMetacat.error("Could not get output stream from response", e); - } - serializeException(be, out); - } catch (Exception e) { - // report Exceptions as clearly and generically as possible - logMetacat.error(e.getClass() + ": " + e.getMessage(), e); - try { - out = response.getOutputStream(); - } catch (IOException ioe) { - logMetacat.error("Could not get output stream from response", ioe); - } - ServiceFailure se = new ServiceFailure("0000", e.getMessage()); - serializeException(se, out); - } - } - - /* - * Handle the solr query sent by the http post method - */ - private void doPostQuery(String engine) { - OutputStream out = null; - try { - // NOTE: we set the session explicitly for the MNode instance since these methods do not provide a parameter - collectMultipartParams(); - MNodeService mnode = MNodeService.getInstance(request); - if (multipartparams == null || multipartparams.isEmpty()) { - throw new InvalidRequest("2823", "The request doesn't have any query information by the HTTP POST method."); - } - HashMap params = new HashMap(); - for (String key : multipartparams.keySet()) { - List values = multipartparams.get(key); - logMetacat.debug("MNResourceHandler.doPostQuery -the key " + key + " has the value " + values); - if (values != null) { - String[] arrayValues = values.toArray(new String[0]); - params.put(key, arrayValues); - } - } - mnode.setSession(session); - InputStream stream = mnode.postQuery(session, engine, params); - // set the content-type if we have it from the implementation - if (stream instanceof ContentTypeInputStream) { - response.setContentType(((ContentTypeInputStream) stream).getContentType()); - } - response.setStatus(200); - out = response.getOutputStream(); - // write the results to the output stream - IOUtils.copyLarge(stream, out); - IOUtils.closeQuietly(out); - return; - } catch (BaseException be) { - // report Exceptions as clearly as possible - try { - out = response.getOutputStream(); - } catch (IOException e) { - logMetacat.error("Could not get output stream from response", e); - } - serializeException(be, out); - } catch (Exception e) { - // report Exceptions as clearly and generically as possible - logMetacat.error(e.getClass() + ": " + e.getMessage(), e); - try { - out = response.getOutputStream(); - } catch (IOException ioe) { - logMetacat.error("Could not get output stream from response", ioe); - } - ServiceFailure se = new ServiceFailure("0000", e.getMessage()); - serializeException(se, out); - } - } - - private void doViews(String format, String pid) { - - OutputStream out = null; - MNodeService mnode = MNodeService.getInstance(request); - - try { - // get a list of views - if (pid != null) { - long start = System.currentTimeMillis(); - Identifier identifier = new Identifier(); - identifier.setValue(pid); - InputStream stream = null; - try { - stream = mnode.view(session, format, identifier); - // set the content-type if we have it from the implementation - if (stream instanceof ContentTypeInputStream) { - response.setContentType(((ContentTypeInputStream) stream).getContentType()); - } - response.setStatus(200); - out = response.getOutputStream(); - // write the results to the output stream - IOUtils.copyLarge(stream, out); - } finally { - if (stream != null) { - IOUtils.closeQuietly(stream); - } - } - long end = System.currentTimeMillis(); - IOUtils.closeQuietly(out); - logMetacat.info(Settings.PERFORMANCELOG + pid + Settings.PERFORMANCELOG_VIEW_METHOD + " Total view method" + Settings.PERFORMANCELOG_DURATION + (end - start) / 1000); - return; - } else { - // TODO: list the registered views - //BaseException ni = new NotImplemented("9999", "MN.listViews() is not implemented at this node"); - //throw ni; - OptionList list = mnode.listViews(session); - - response.setContentType("text/xml"); - response.setStatus(200); - TypeMarshaller.marshalTypeToOutputStream(list, response.getOutputStream()); - IOUtils.closeQuietly(response.getOutputStream()); - } - - - } catch (BaseException be) { - // report Exceptions as clearly as possible - try { - out = response.getOutputStream(); - } catch (IOException e) { - logMetacat.error("Could not get output stream from response", e); - } - serializeException(be, out); - } catch (Exception e) { - // report Exceptions as clearly and generically as possible - logMetacat.error(e.getClass() + ": " + e.getMessage(), e); - try { - out = response.getOutputStream(); - } catch (IOException ioe) { - logMetacat.error("Could not get output stream from response", ioe); - } - ServiceFailure se = new ServiceFailure("0000", e.getMessage()); - serializeException(se, out); - } - } - - /** - * Handles notification of system metadata changes for the given identifier - * - * @param id the identifier for the object - * @throws InvalidToken - * @throws InvalidRequest - * @throws NotAuthorized - * @throws ServiceFailure - * @throws NotImplemented - */ - private void systemMetadataChanged() - throws NotImplemented, ServiceFailure, NotAuthorized, InvalidRequest, - InvalidToken { - - ReadOnlyChecker checker = new ReadOnlyChecker(); - boolean isReadOnlyMode = checker.isReadOnly(); - if (isReadOnlyMode) { - throw new ServiceFailure("1333", ReadOnlyChecker.DATAONEERROR); - } - - //final long serialVersion = 0L; - String serialVersionStr = null; - String dateSysMetaLastModifiedStr = null; - - // mkae sure we have the multipart params - try { - initMultipartParams(); - } catch (Exception e1) { - throw new ServiceFailure("1333", "Could not collect the multipart params for the request"); - } - - // get the pid - String id = null; - try { - id = multipartparams.get("pid").get(0); - } catch (NullPointerException e) { - String msg = "The 'pid' must be provided as a parameter and was not."; - logMetacat.error(msg); - throw new InvalidRequest("1334", msg); - } - final Identifier pid = new Identifier(); - pid.setValue(id); - - // get the serialVersion - try { - serialVersionStr = multipartparams.get("serialVersion").get(0); - } catch (NullPointerException e) { - String msg = "The 'serialVersion' must be provided as a parameter and was not."; - logMetacat.error(msg); - throw new InvalidRequest("1334", msg); - - } - - final long serialVersion = (new Long(serialVersionStr)).longValue(); - - // get the dateSysMetaLastModified - try { - dateSysMetaLastModifiedStr = multipartparams.get("dateSysMetaLastModified").get(0); - - - } catch (NullPointerException e) { - String msg = - "The 'dateSysMetaLastModified' must be provided as a " + - "parameter and was not, or was an invalid representation of the timestamp."; - logMetacat.error(msg); - throw new InvalidRequest("1334", msg); - - } - final Date dateSysMetaLastModified = DateTimeMarshaller.deserializeDateToUTC(dateSysMetaLastModifiedStr); - - // check authorization before sending to implementation - D1AuthHelper authDel = new D1AuthHelper(request, pid, "1331", "????"); - authDel.doAdminAuthorization(session); + // MN-specific API Resources + protected static final String RESOURCE_MONITOR = "monitor"; + protected static final String RESOURCE_REPLICATE = "replicate"; + protected static final String RESOURCE_REPLICAS = "replica"; + protected static final String RESOURCE_NODE = "node"; + protected static final String RESOURCE_ERROR = "error"; + protected static final String RESOURCE_META_CHANGED = "dirtySystemMetadata"; + protected static final String RESOURCE_GENERATE_ID = "generate"; + protected static final String RESOURCE_PUBLISH = "publish"; + protected static final String RESOURCE_PACKAGE = "packages"; + protected static final String RESOURCE_TOKEN = "token"; + protected static final String RESOURCE_WHOAMI = "whoami"; + + + // shared executor + private static ExecutorService executor = null; + + static { + // use a shared executor service with nThreads == one less than available processors + int availableProcessors = Runtime.getRuntime().availableProcessors(); + int nThreads = availableProcessors * 1; + nThreads--; + nThreads = Math.max(1, nThreads); + executor = Executors.newFixedThreadPool(nThreads); + } + + /** + * Initializes new instance by setting servlet context,request and response + */ + public MNResourceHandler(ServletContext servletContext, + HttpServletRequest request, HttpServletResponse response) { + super(servletContext, request, response); + logMetacat = LogFactory.getLog(MNResourceHandler.class); + } + + @Override + protected boolean isD1Enabled() { + + boolean enabled = false; + try { + enabled = Boolean.parseBoolean(PropertyService.getProperty("dataone.mn.services.enabled")); + } catch (PropertyNotFoundException e) { + logMetacat.error("Could not check if DataONE is enabled: " + e.getMessage()); + } + + return enabled; + } + + /** + * This function is called from REST API servlet and handles each request to the servlet + * + * @param httpVerb (GET, POST, PUT or DELETE) + */ + @Override + public void handle(byte httpVerb) { + // prepare the handler + super.handle(httpVerb); + + try { + + // only service requests if we have D1 configured + if (!isD1Enabled()) { + ServiceFailure se = new ServiceFailure("0000", "DataONE services are not enabled on this node"); + serializeException(se, response.getOutputStream()); + return; + } + + // get the resource + String resource = request.getPathInfo(); + resource = resource.substring(resource.indexOf("/") + 1); + + // default to node info + if (resource.equals("")) { + resource = RESOURCE_NODE; + } + + // get the rest of the path info + String extra = null; + + logMetacat.info("MNResourceHandler.handle - V2 handling verb " + httpVerb + " request with resource '" + resource + "'"); + logMetacat.debug("resource: '" + resource + "'"); + boolean status = false; + + if (resource != null) { + + if (resource.startsWith(RESOURCE_NODE)) { + // node response + node(); + status = true; + } else if (resource.startsWith(RESOURCE_TOKEN)) { + logMetacat.debug("Using resource 'token'"); + // get + if (httpVerb == GET) { + // after the command + getToken(); + status = true; + } + + } else if (resource.startsWith(RESOURCE_WHOAMI)) { + logMetacat.debug("Using resource 'whoami'"); + // get + if (httpVerb == GET) { + // after the command + whoami(); + status = true; + } + + } else if (resource.startsWith(RESOURCE_IS_AUTHORIZED)) { + if (httpVerb == GET) { + // after the command + extra = parseTrailing(resource, RESOURCE_IS_AUTHORIZED); + extra = decode(extra); + // check the access rules + isAuthorized(extra); + status = true; + logMetacat.debug("done getting access"); + } + } else if (resource.startsWith(RESOURCE_META)) { + logMetacat.debug("Using resource 'meta'"); + // get + if (httpVerb == GET) { + logMetacat.debug("Using resource 'meta' for GET"); + // after the command + extra = parseTrailing(resource, RESOURCE_META); + extra = decode(extra); + getSystemMetadataObject(extra); + status = true; + } else if (httpVerb == PUT) { + logMetacat.debug("Using resource 'meta' for PUT"); + updateSystemMetadata(); + status = true; + } + + } else if (resource.startsWith(RESOURCE_OBJECTS)) { + logMetacat.debug("Using resource 'object'"); + // after the command + extra = parseTrailing(resource, RESOURCE_OBJECTS); + extra = decode(extra); + logMetacat.debug("objectId: " + extra); + logMetacat.debug("verb:" + httpVerb); + + if (httpVerb == GET) { + getObject(extra); + status = true; + } else if (httpVerb == POST) { + // part of the params, not the URL + putObject(null, FUNCTION_NAME_INSERT); + status = true; + } else if (httpVerb == PUT) { + putObject(extra, FUNCTION_NAME_UPDATE); + status = true; + } else if (httpVerb == DELETE) { + deleteObject(extra); + status = true; + } else if (httpVerb == HEAD) { + describeObject(extra); + status = true; + } + + } else if (resource.startsWith(RESOURCE_LOG)) { + logMetacat.debug("Using resource 'log'"); + // handle log events + if (httpVerb == GET) { + getLog(); + status = true; + } + } else if (resource.startsWith(Constants.RESOURCE_ARCHIVE)) { + logMetacat.debug("Using resource " + Constants.RESOURCE_ARCHIVE); + // handle archive events + if (httpVerb == PUT) { + extra = parseTrailing(resource, Constants.RESOURCE_ARCHIVE); + extra = decode(extra); + archive(extra); + status = true; + } + } else if (resource.startsWith(Constants.RESOURCE_CHECKSUM)) { + logMetacat.debug("Using resource 'checksum'"); + // handle checksum requests + if (httpVerb == GET) { + // after the command + extra = parseTrailing(resource, Constants.RESOURCE_CHECKSUM); + extra = decode(extra); + checksum(extra); + status = true; + } + } else if (resource.startsWith(RESOURCE_MONITOR)) { + // there are various parts to monitoring + if (httpVerb == GET) { + // after the command + extra = parseTrailing(resource, RESOURCE_MONITOR); + extra = decode(extra); + // ping + if (extra.toLowerCase().equals("ping")) { + logMetacat.debug("processing ping request"); + Date result = MNodeService.getInstance(request).ping(); + // TODO: send to output + status = true; + + } else if (extra.toLowerCase().equals("status")) { + logMetacat.debug("processing status request"); + getStatus(); + status = true; + } + + } + } else if (resource.startsWith(RESOURCE_REPLICATE)) { + if (httpVerb == POST) { + logMetacat.debug("processing replicate request"); + replicate(); + status = true; + } + } else if (resource.startsWith(RESOURCE_ERROR)) { + // sync error + if (httpVerb == POST) { + syncError(); + status = true; + } + } else if (resource.startsWith(RESOURCE_META_CHANGED)) { + // system metadata changed + if (httpVerb == POST) { + systemMetadataChanged(); + status = true; + } + } else if (resource.startsWith(RESOURCE_REPLICAS)) { + // get replica + if (httpVerb == GET) { + extra = parseTrailing(resource, RESOURCE_REPLICAS); + extra = decode(extra); + getReplica(extra); + status = true; + } + } else if (resource.startsWith(RESOURCE_QUERY)) { + logMetacat.debug("Using resource " + RESOURCE_QUERY); + // after the command + extra = parseTrailing(resource, RESOURCE_QUERY); + logMetacat.debug("query extra: " + extra); + + String engine = null; + String query = null; + + if (extra != null) { + // get the engine + int engineIndex = extra.length(); + if (extra.indexOf("/") > -1) { + engineIndex = extra.indexOf("/"); + } + engine = extra.substring(0, engineIndex); + engine = decode(engine); + logMetacat.debug("query engine: " + engine); + + // check the query string first + query = request.getQueryString(); + + // if null, look at the whole endpoint + if (query == null) { + // get the query if it is there + query = extra.substring(engineIndex, extra.length()); + if (query != null && query.length() == 0) { + query = null; + } else { + if (query.startsWith("/")) { + query = query.substring(1); + } + } + } + query = decode(query); + logMetacat.debug("query: " + query); + + } + logMetacat.debug("verb:" + httpVerb); + if (httpVerb == GET) { + doQuery(engine, query); + status = true; + } else if (httpVerb == POST) { + doPostQuery(engine); + status = true; + } + } else if (resource.startsWith(RESOURCE_GENERATE_ID)) { + // generate an id + if (httpVerb == POST) { + generateIdentifier(); + status = true; + } + } else if (resource.startsWith(RESOURCE_PUBLISH)) { + logMetacat.debug("Using resource: " + RESOURCE_PUBLISH); + // PUT + if (httpVerb == PUT) { + // after the command + extra = parseTrailing(resource, RESOURCE_PUBLISH); + extra = decode(extra); + publish(extra); + status = true; + } + } else if (resource.startsWith(RESOURCE_PACKAGE)) { + logMetacat.debug("Using resource: " + RESOURCE_PACKAGE); + // after the command + extra = parseTrailing(resource, RESOURCE_PACKAGE); + + String format = null; + String pid = null; + + if (extra != null) { + // get the format + int formatIndex = extra.length(); + if (extra.indexOf("/") > -1) { + formatIndex = extra.indexOf("/"); + } + format = extra.substring(0, formatIndex); + format = decode(format); + logMetacat.debug("package format: " + format); + + // get the pid if it is there + pid = extra.substring(formatIndex, extra.length()); + if (pid != null && pid.length() == 0) { + pid = null; + } else { + if (pid.startsWith("/")) { + pid = pid.substring(1); + } + } + pid = decode(pid); + logMetacat.debug("pid: " + pid); + + } + + // get + if (httpVerb == GET) { + + getPackage(format, pid); + status = true; + } + } else if (resource.startsWith(RESOURCE_VIEWS)) { + logMetacat.debug("Using resource " + RESOURCE_VIEWS); + // after the command + extra = parseTrailing(resource, RESOURCE_VIEWS); + logMetacat.debug("view extra: " + extra); + + String format = null; + String pid = null; + + if (extra != null) { + // get the format + int formatIndex = extra.length(); + if (extra.indexOf("/") > -1) { + formatIndex = extra.indexOf("/"); + } + format = extra.substring(0, formatIndex); + format = decode(format); + logMetacat.debug("view format: " + format); + + // get the pid if it is there + pid = extra.substring(formatIndex, extra.length()); + if (pid != null && pid.length() == 0) { + pid = null; + } else { + if (pid.startsWith("/")) { + pid = pid.substring(1); + } + } + pid = decode(pid); + logMetacat.debug("pid: " + pid); + + } + logMetacat.debug("verb:" + httpVerb); + if (httpVerb == GET) { + doViews(format, pid); + status = true; + } + } else { + throw new InvalidRequest("0000", "No resource matched for " + resource); + } + + if (!status) { + throw new ServiceFailure("0000", "Unknown error, status = " + status); + } + } else { + throw new InvalidRequest("0000", "No resource matched for " + resource); + } + } catch (BaseException be) { + // report Exceptions as clearly as possible + OutputStream out = null; + try { + out = response.getOutputStream(); + } catch (IOException e) { + logMetacat.error("Could not get output stream from response", e); + } + serializeException(be, out); + } catch (Exception e) { + // report Exceptions as clearly and generically as possible + logMetacat.error(e.getClass() + ": " + e.getMessage(), e); + OutputStream out = null; + try { + out = response.getOutputStream(); + } catch (IOException ioe) { + logMetacat.error("Could not get output stream from response", ioe); + } + ServiceFailure se = new ServiceFailure("0000", e.getMessage()); + serializeException(se, out); + } + } + + + private void doQuery(String engine, String query) { + + OutputStream out = null; + + try { + // NOTE: we set the session explicitly for the MNode instance since these methods do not provide a parameter + if (engine == null) { + // just looking for list of engines + MNodeService mnode = MNodeService.getInstance(request); + mnode.setSession(session); + QueryEngineList qel = mnode.listQueryEngines(session); + response.setContentType("text/xml"); + response.setStatus(200); + out = response.getOutputStream(); + TypeMarshaller.marshalTypeToOutputStream(qel, out); + IOUtils.closeQuietly(out); + return; + } else { + if (query != null) { + long start = System.currentTimeMillis(); + MNodeService mnode = MNodeService.getInstance(request); + mnode.setSession(session); + InputStream stream = mnode.query(session, engine, query); + + // set the content-type if we have it from the implementation + if (stream instanceof ContentTypeInputStream) { + //response.setContentType("application/octet-stream"); + //response.setContentType("text/xml"); + response.setContentType(((ContentTypeInputStream) stream).getContentType()); + } + response.setStatus(200); + out = response.getOutputStream(); + // write the results to the output stream + IOUtils.copyLarge(stream, out); + long end = System.currentTimeMillis(); + logMetacat.info(Settings.PERFORMANCELOG + Settings.PERFORMANCELOG_QUERY_METHOD + query + " Total query method" + Settings.PERFORMANCELOG_DURATION + (end - start) / 1000); + IOUtils.closeQuietly(out); + return; + } else { + MNodeService mnode = MNodeService.getInstance(request); + mnode.setSession(session); + QueryEngineDescription qed = mnode.getQueryEngineDescription(session, engine); + response.setContentType("text/xml"); + response.setStatus(200); + out = response.getOutputStream(); + TypeMarshaller.marshalTypeToOutputStream(qed, out); + IOUtils.closeQuietly(out); + return; + } + } + + + } catch (BaseException be) { + // report Exceptions as clearly as possible + try { + out = response.getOutputStream(); + } catch (IOException e) { + logMetacat.error("Could not get output stream from response", e); + } + serializeException(be, out); + } catch (Exception e) { + // report Exceptions as clearly and generically as possible + logMetacat.error(e.getClass() + ": " + e.getMessage(), e); + try { + out = response.getOutputStream(); + } catch (IOException ioe) { + logMetacat.error("Could not get output stream from response", ioe); + } + ServiceFailure se = new ServiceFailure("0000", e.getMessage()); + serializeException(se, out); + } + } + + /* + * Handle the solr query sent by the http post method + */ + private void doPostQuery(String engine) { + OutputStream out = null; + try { + // NOTE: we set the session explicitly for the MNode instance since these methods do not provide a parameter + collectMultipartParams(); + MNodeService mnode = MNodeService.getInstance(request); + if (multipartparams == null || multipartparams.isEmpty()) { + throw new InvalidRequest("2823", "The request doesn't have any query information by the HTTP POST method."); + } + HashMap params = new HashMap(); + for (String key : multipartparams.keySet()) { + List values = multipartparams.get(key); + logMetacat.debug("MNResourceHandler.doPostQuery -the key " + key + " has the value " + values); + if (values != null) { + String[] arrayValues = values.toArray(new String[0]); + params.put(key, arrayValues); + } + } + mnode.setSession(session); + InputStream stream = mnode.postQuery(session, engine, params); + // set the content-type if we have it from the implementation + if (stream instanceof ContentTypeInputStream) { + response.setContentType(((ContentTypeInputStream) stream).getContentType()); + } + response.setStatus(200); + out = response.getOutputStream(); + // write the results to the output stream + IOUtils.copyLarge(stream, out); + IOUtils.closeQuietly(out); + return; + } catch (BaseException be) { + // report Exceptions as clearly as possible + try { + out = response.getOutputStream(); + } catch (IOException e) { + logMetacat.error("Could not get output stream from response", e); + } + serializeException(be, out); + } catch (Exception e) { + // report Exceptions as clearly and generically as possible + logMetacat.error(e.getClass() + ": " + e.getMessage(), e); + try { + out = response.getOutputStream(); + } catch (IOException ioe) { + logMetacat.error("Could not get output stream from response", ioe); + } + ServiceFailure se = new ServiceFailure("0000", e.getMessage()); + serializeException(se, out); + } + } + + private void doViews(String format, String pid) { + + OutputStream out = null; + MNodeService mnode = MNodeService.getInstance(request); + + try { + // get a list of views + if (pid != null) { + long start = System.currentTimeMillis(); + Identifier identifier = new Identifier(); + identifier.setValue(pid); + InputStream stream = null; + try { + stream = mnode.view(session, format, identifier); + // set the content-type if we have it from the implementation + if (stream instanceof ContentTypeInputStream) { + response.setContentType(((ContentTypeInputStream) stream).getContentType()); + } + response.setStatus(200); + out = response.getOutputStream(); + // write the results to the output stream + IOUtils.copyLarge(stream, out); + } finally { + if (stream != null) { + IOUtils.closeQuietly(stream); + } + } + long end = System.currentTimeMillis(); + IOUtils.closeQuietly(out); + logMetacat.info(Settings.PERFORMANCELOG + pid + Settings.PERFORMANCELOG_VIEW_METHOD + " Total view method" + Settings.PERFORMANCELOG_DURATION + (end - start) / 1000); + return; + } else { + // TODO: list the registered views + //BaseException ni = new NotImplemented("9999", "MN.listViews() is not implemented at this node"); + //throw ni; + OptionList list = mnode.listViews(session); + + response.setContentType("text/xml"); + response.setStatus(200); + TypeMarshaller.marshalTypeToOutputStream(list, response.getOutputStream()); + IOUtils.closeQuietly(response.getOutputStream()); + } + + + } catch (BaseException be) { + // report Exceptions as clearly as possible + try { + out = response.getOutputStream(); + } catch (IOException e) { + logMetacat.error("Could not get output stream from response", e); + } + serializeException(be, out); + } catch (Exception e) { + // report Exceptions as clearly and generically as possible + logMetacat.error(e.getClass() + ": " + e.getMessage(), e); + try { + out = response.getOutputStream(); + } catch (IOException ioe) { + logMetacat.error("Could not get output stream from response", ioe); + } + ServiceFailure se = new ServiceFailure("0000", e.getMessage()); + serializeException(se, out); + } + } + + /** + * Handles notification of system metadata changes for the given identifier + * + * @param id the identifier for the object + * @throws InvalidToken + * @throws InvalidRequest + * @throws NotAuthorized + * @throws ServiceFailure + * @throws NotImplemented + */ + private void systemMetadataChanged() + throws NotImplemented, ServiceFailure, NotAuthorized, InvalidRequest, + InvalidToken { + + ReadOnlyChecker checker = new ReadOnlyChecker(); + boolean isReadOnlyMode = checker.isReadOnly(); + if (isReadOnlyMode) { + throw new ServiceFailure("1333", ReadOnlyChecker.DATAONEERROR); + } + + //final long serialVersion = 0L; + String serialVersionStr = null; + String dateSysMetaLastModifiedStr = null; + + // mkae sure we have the multipart params + try { + initMultipartParams(); + } catch (Exception e1) { + throw new ServiceFailure("1333", "Could not collect the multipart params for the request"); + } + + // get the pid + String id = null; + try { + id = multipartparams.get("pid").get(0); + } catch (NullPointerException e) { + String msg = "The 'pid' must be provided as a parameter and was not."; + logMetacat.error(msg); + throw new InvalidRequest("1334", msg); + } + final Identifier pid = new Identifier(); + pid.setValue(id); + + // get the serialVersion + try { + serialVersionStr = multipartparams.get("serialVersion").get(0); + } catch (NullPointerException e) { + String msg = "The 'serialVersion' must be provided as a parameter and was not."; + logMetacat.error(msg); + throw new InvalidRequest("1334", msg); + + } + + final long serialVersion = (new Long(serialVersionStr)).longValue(); + + // get the dateSysMetaLastModified + try { + dateSysMetaLastModifiedStr = multipartparams.get("dateSysMetaLastModified").get(0); + + + } catch (NullPointerException e) { + String msg = + "The 'dateSysMetaLastModified' must be provided as a " + + "parameter and was not, or was an invalid representation of the timestamp."; + logMetacat.error(msg); + throw new InvalidRequest("1334", msg); + + } + final Date dateSysMetaLastModified = DateTimeMarshaller.deserializeDateToUTC(dateSysMetaLastModifiedStr); + + // check authorization before sending to implementation + D1AuthHelper authDel = new D1AuthHelper(request, pid, "1331", "????"); + authDel.doAdminAuthorization(session); // boolean authorized = MNodeService.getInstance(request).isAdminAuthorized(session); // if (!authorized) { // String msg = "User is not authorized to call systemMetadataChanged"; @@ -821,726 +821,726 @@ private void systemMetadataChanged() // throw failure; // } - // run it in a thread to avoid connection timeout - final String ipAddress = request.getRemoteAddr(); - final String userAgent = request.getHeader("User-Agent"); - Runnable runner = new Runnable() { - @Override - public void run() { - try { - // call the service - MNodeService.getInstance(request, ipAddress, userAgent).systemMetadataChanged(session, pid, serialVersion, dateSysMetaLastModified); - } catch (Exception e) { - logMetacat.error("Error running replication: " + e.getMessage(), e); - throw new RuntimeException(e.getMessage(), e); - } - } - }; - // submit the task, and that's it - executor.submit(runner); - - // thread was started, so we return success - response.setStatus(200); - } - - /** - * Handles identifier generation calls - * - * @throws InvalidRequest - * @throws NotImplemented - * @throws NotAuthorized - * @throws ServiceFailure - * @throws InvalidToken - * @throws IOException - * @throws MarshallingException - */ - private void generateIdentifier() throws InvalidToken, ServiceFailure, NotAuthorized, NotImplemented, InvalidRequest, IOException, MarshallingException { - - // make sure we have the multipart params - try { - initMultipartParams(); - } catch (Exception e1) { - throw new ServiceFailure("1333", "Could not collect the multipart params for the request"); - } - - // get the scheme - String scheme = null; - try { - scheme = multipartparams.get("scheme").get(0); - } catch (NullPointerException e) { - String msg = "The 'scheme' parameter was not provided, using default"; - logMetacat.warn(msg); - } - - // get the fragment - String fragment = null; - try { - fragment = multipartparams.get("fragment").get(0); - } catch (NullPointerException e) { - String msg = "The 'fragment' parameter was not provided, using default"; - logMetacat.warn(msg); - } - - // call the service - Identifier identifier = MNodeService.getInstance(request).generateIdentifier(session, scheme, fragment); - response.setStatus(200); - response.setContentType("text/xml"); - OutputStream out = response.getOutputStream(); - TypeMarshaller.marshalTypeToOutputStream(identifier, out); - IOUtils.closeQuietly(out); - } - - /** - * Checks the access policy - * - * @param id - * @return - * @throws ServiceFailure - * @throws InvalidToken - * @throws NotFound - * @throws NotAuthorized - * @throws NotImplemented - * @throws InvalidRequest - */ - private boolean isAuthorized(String id) throws ServiceFailure, InvalidToken, NotFound, NotAuthorized, NotImplemented, InvalidRequest { - Identifier pid = new Identifier(); - pid.setValue(id); - Permission permission = null; - try { - String perm = params.get("action")[0]; - permission = Permission.convert(perm); - } catch (Exception e) { - logMetacat.warn("No permission specified"); - } - boolean result = MNodeService.getInstance(request).isAuthorized(session, pid, permission); - response.setStatus(200); - response.setContentType("text/xml"); - return result; - } - - private void getToken() throws Exception { - - if (this.session != null) { - String userId = this.session.getSubject().getValue(); - String fullName = null; - try { - Person person = this.session.getSubjectInfo().getPerson(0); - fullName = person.getGivenName(0) + " " + person.getFamilyName(); - } catch (Exception e) { - logMetacat.warn(e.getMessage(), e); - } - - String token = null; - token = TokenGenerator.getInstance().getJWT(userId, fullName); - - response.setStatus(200); - response.setContentType("text/plain"); - OutputStream out = response.getOutputStream(); - out.write(token.getBytes(MetaCatServlet.DEFAULT_ENCODING)); - out.close(); - } else { - response.setStatus(401); - response.setContentType("text/plain"); - OutputStream out = response.getOutputStream(); - out.write("No session information found".getBytes(MetaCatServlet.DEFAULT_ENCODING)); - out.close(); - } - - } - - private void whoami() throws Exception { - - if (this.session != null) { - Subject subject = this.session.getSubject(); - SubjectInfo subjectInfo = null; - try { - subjectInfo = this.session.getSubjectInfo(); - } catch (Exception e) { - logMetacat.warn(e.getMessage(), e); - } - - response.setStatus(200); - response.setContentType("text/plain"); - OutputStream out = response.getOutputStream(); - - if (subjectInfo != null) { - TypeMarshaller.marshalTypeToOutputStream(subjectInfo, out); - } else { - TypeMarshaller.marshalTypeToOutputStream(subject, out); - } - - out.close(); - } else { - response.setStatus(401); - response.setContentType("text/plain"); - OutputStream out = response.getOutputStream(); - out.write("No session information found".getBytes(MetaCatServlet.DEFAULT_ENCODING)); - out.close(); - } - - } - - /** - * Get the status of the system. Now we only support to get the size of the index queue. - */ - private void getStatus() throws IOException, NotAuthorized, ServiceFailure { - InputStream result = MNodeService.getInstance(request).getStatus(this.session); - response.setStatus(200); - response.setContentType("text/xml"); - OutputStream out = response.getOutputStream(); - out = response.getOutputStream(); - IOUtils.copy(result, out); - IOUtils.closeQuietly(out); - //IOUtils.copyLarge(result, out); - } - - /** - * Processes failed synchronization message - * - * @throws NotImplemented - * @throws ServiceFailure - * @throws NotAuthorized - * @throws InvalidRequest - * @throws MarshallingException - * @throws IllegalAccessException - * @throws InstantiationException - * @throws IOException - */ - private void syncError() throws NotImplemented, ServiceFailure, NotAuthorized, InvalidRequest, MarshallingException, IOException, InstantiationException, IllegalAccessException { - SynchronizationFailed syncFailed = null; - try { - syncFailed = collectSynchronizationFailed(); - } catch (ParserConfigurationException e) { - throw new ServiceFailure("2161", e.getMessage()); - } catch (SAXException e) { - throw new ServiceFailure("2161", e.getMessage()); - } - - MNodeService.getInstance(request).synchronizationFailed(session, syncFailed); - } - - /** - * Calculate the checksum - * - * @throws NotImplemented - * @throws MarshallingException - * @throws IOException - * @throws InvalidToken - * @throws ServiceFailure - * @throws NotAuthorized - * @throws NotFound - * @throws InvalidRequest - */ - private void checksum(String pid) throws NotImplemented, MarshallingException, IOException, InvalidToken, ServiceFailure, NotAuthorized, NotFound, InvalidRequest { - String checksumAlgorithm = "MD5"; - try { - checksumAlgorithm = PropertyService.getProperty("dataone.checksumAlgorithm.default"); - } catch (Exception e) { - logMetacat.warn("Could not lookup configured default checksum algorithm, using: " + checksumAlgorithm); - } - - Identifier pidid = new Identifier(); - pidid.setValue(pid); - try { - checksumAlgorithm = params.get("checksumAlgorithm")[0]; - } catch (Exception e) { - //do nothing. use the default - logMetacat.warn("No algorithm specified, using default: " + checksumAlgorithm); - } - logMetacat.debug("getting checksum for object " + pid + " with algorithm " + checksumAlgorithm); - - Checksum c = MNodeService.getInstance(request).getChecksum(session, pidid, checksumAlgorithm); - logMetacat.debug("got checksum " + c.getValue()); - response.setStatus(200); - logMetacat.debug("serializing response"); - TypeMarshaller.marshalTypeToOutputStream(c, response.getOutputStream()); - logMetacat.debug("done serializing response."); - IOUtils.closeQuietly(response.getOutputStream()); - - } - - /** - * handle the replicate action for MN - * - * @throws MarshallingException - * @throws FileUploadException - * @throws IOException - * @throws InvalidRequest - * @throws ServiceFailure - * @throws UnsupportedType - * @throws InsufficientResources - * @throws NotAuthorized - * @throws NotImplemented - * @throws IllegalAccessException - * @throws InstantiationException - * @throws InvalidToken - */ - private void replicate() - throws ServiceFailure, InvalidRequest, IOException, FileUploadException, - MarshallingException, NotImplemented, NotAuthorized, InsufficientResources, - UnsupportedType, InstantiationException, IllegalAccessException, InvalidToken { - - logMetacat.debug("in POST replicate()"); - ReadOnlyChecker checker = new ReadOnlyChecker(); - boolean isReadOnlyMode = checker.isReadOnly(); - if (isReadOnlyMode) { - throw new ServiceFailure("2151", ReadOnlyChecker.DATAONEERROR); - } - - // somewhat unorthodox, but the call is asynchronous and we'd like to return this info sooner - boolean allowed = false; - if (session == null) { - String msg = "No session was provided."; - NotAuthorized failure = new NotAuthorized("2152", msg); - throw failure; - } else { - // TODO: should we refactore replicate() in MNodeservice to not replicate, it would avoid a possible second listNodes call... - D1AuthHelper authDel = new D1AuthHelper(request, null, "2152", "????"); - authDel.doAdminAuthorization(session); + // run it in a thread to avoid connection timeout + final String ipAddress = request.getRemoteAddr(); + final String userAgent = request.getHeader("User-Agent"); + Runnable runner = new Runnable() { + @Override + public void run() { + try { + // call the service + MNodeService.getInstance(request, ipAddress, userAgent).systemMetadataChanged(session, pid, serialVersion, dateSysMetaLastModified); + } catch (Exception e) { + logMetacat.error("Error running replication: " + e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); + } + } + }; + // submit the task, and that's it + executor.submit(runner); + + // thread was started, so we return success + response.setStatus(200); + } + + /** + * Handles identifier generation calls + * + * @throws InvalidRequest + * @throws NotImplemented + * @throws NotAuthorized + * @throws ServiceFailure + * @throws InvalidToken + * @throws IOException + * @throws MarshallingException + */ + private void generateIdentifier() throws InvalidToken, ServiceFailure, NotAuthorized, NotImplemented, InvalidRequest, IOException, MarshallingException { + + // make sure we have the multipart params + try { + initMultipartParams(); + } catch (Exception e1) { + throw new ServiceFailure("1333", "Could not collect the multipart params for the request"); + } + + // get the scheme + String scheme = null; + try { + scheme = multipartparams.get("scheme").get(0); + } catch (NullPointerException e) { + String msg = "The 'scheme' parameter was not provided, using default"; + logMetacat.warn(msg); + } + + // get the fragment + String fragment = null; + try { + fragment = multipartparams.get("fragment").get(0); + } catch (NullPointerException e) { + String msg = "The 'fragment' parameter was not provided, using default"; + logMetacat.warn(msg); + } + + // call the service + Identifier identifier = MNodeService.getInstance(request).generateIdentifier(session, scheme, fragment); + response.setStatus(200); + response.setContentType("text/xml"); + OutputStream out = response.getOutputStream(); + TypeMarshaller.marshalTypeToOutputStream(identifier, out); + IOUtils.closeQuietly(out); + } + + /** + * Checks the access policy + * + * @param id + * @return + * @throws ServiceFailure + * @throws InvalidToken + * @throws NotFound + * @throws NotAuthorized + * @throws NotImplemented + * @throws InvalidRequest + */ + private boolean isAuthorized(String id) throws ServiceFailure, InvalidToken, NotFound, NotAuthorized, NotImplemented, InvalidRequest { + Identifier pid = new Identifier(); + pid.setValue(id); + Permission permission = null; + try { + String perm = params.get("action")[0]; + permission = Permission.convert(perm); + } catch (Exception e) { + logMetacat.warn("No permission specified"); + } + boolean result = MNodeService.getInstance(request).isAuthorized(session, pid, permission); + response.setStatus(200); + response.setContentType("text/xml"); + return result; + } + + private void getToken() throws Exception { + + if (this.session != null) { + String userId = this.session.getSubject().getValue(); + String fullName = null; + try { + Person person = this.session.getSubjectInfo().getPerson(0); + fullName = person.getGivenName(0) + " " + person.getFamilyName(); + } catch (Exception e) { + logMetacat.warn(e.getMessage(), e); + } + + String token = null; + token = TokenGenerator.getInstance().getJWT(userId, fullName); + + response.setStatus(200); + response.setContentType("text/plain"); + OutputStream out = response.getOutputStream(); + out.write(token.getBytes(MetaCatServlet.DEFAULT_ENCODING)); + out.close(); + } else { + response.setStatus(401); + response.setContentType("text/plain"); + OutputStream out = response.getOutputStream(); + out.write("No session information found".getBytes(MetaCatServlet.DEFAULT_ENCODING)); + out.close(); + } + + } + + private void whoami() throws Exception { + + if (this.session != null) { + Subject subject = this.session.getSubject(); + SubjectInfo subjectInfo = null; + try { + subjectInfo = this.session.getSubjectInfo(); + } catch (Exception e) { + logMetacat.warn(e.getMessage(), e); + } + + response.setStatus(200); + response.setContentType("text/plain"); + OutputStream out = response.getOutputStream(); + + if (subjectInfo != null) { + TypeMarshaller.marshalTypeToOutputStream(subjectInfo, out); + } else { + TypeMarshaller.marshalTypeToOutputStream(subject, out); + } + + out.close(); + } else { + response.setStatus(401); + response.setContentType("text/plain"); + OutputStream out = response.getOutputStream(); + out.write("No session information found".getBytes(MetaCatServlet.DEFAULT_ENCODING)); + out.close(); + } + + } + + /** + * Get the status of the system. Now we only support to get the size of the index queue. + */ + private void getStatus() throws IOException, NotAuthorized, ServiceFailure { + InputStream result = MNodeService.getInstance(request).getStatus(this.session); + response.setStatus(200); + response.setContentType("text/xml"); + OutputStream out = response.getOutputStream(); + out = response.getOutputStream(); + IOUtils.copy(result, out); + IOUtils.closeQuietly(out); + //IOUtils.copyLarge(result, out); + } + + /** + * Processes failed synchronization message + * + * @throws NotImplemented + * @throws ServiceFailure + * @throws NotAuthorized + * @throws InvalidRequest + * @throws MarshallingException + * @throws IllegalAccessException + * @throws InstantiationException + * @throws IOException + */ + private void syncError() throws NotImplemented, ServiceFailure, NotAuthorized, InvalidRequest, MarshallingException, IOException, InstantiationException, IllegalAccessException { + SynchronizationFailed syncFailed = null; + try { + syncFailed = collectSynchronizationFailed(); + } catch (ParserConfigurationException e) { + throw new ServiceFailure("2161", e.getMessage()); + } catch (SAXException e) { + throw new ServiceFailure("2161", e.getMessage()); + } + + MNodeService.getInstance(request).synchronizationFailed(session, syncFailed); + } + + /** + * Calculate the checksum + * + * @throws NotImplemented + * @throws MarshallingException + * @throws IOException + * @throws InvalidToken + * @throws ServiceFailure + * @throws NotAuthorized + * @throws NotFound + * @throws InvalidRequest + */ + private void checksum(String pid) throws NotImplemented, MarshallingException, IOException, InvalidToken, ServiceFailure, NotAuthorized, NotFound, InvalidRequest { + String checksumAlgorithm = "MD5"; + try { + checksumAlgorithm = PropertyService.getProperty("dataone.checksumAlgorithm.default"); + } catch (Exception e) { + logMetacat.warn("Could not lookup configured default checksum algorithm, using: " + checksumAlgorithm); + } + + Identifier pidid = new Identifier(); + pidid.setValue(pid); + try { + checksumAlgorithm = params.get("checksumAlgorithm")[0]; + } catch (Exception e) { + //do nothing. use the default + logMetacat.warn("No algorithm specified, using default: " + checksumAlgorithm); + } + logMetacat.debug("getting checksum for object " + pid + " with algorithm " + checksumAlgorithm); + + Checksum c = MNodeService.getInstance(request).getChecksum(session, pidid, checksumAlgorithm); + logMetacat.debug("got checksum " + c.getValue()); + response.setStatus(200); + logMetacat.debug("serializing response"); + TypeMarshaller.marshalTypeToOutputStream(c, response.getOutputStream()); + logMetacat.debug("done serializing response."); + IOUtils.closeQuietly(response.getOutputStream()); + + } + + /** + * handle the replicate action for MN + * + * @throws MarshallingException + * @throws FileUploadException + * @throws IOException + * @throws InvalidRequest + * @throws ServiceFailure + * @throws UnsupportedType + * @throws InsufficientResources + * @throws NotAuthorized + * @throws NotImplemented + * @throws IllegalAccessException + * @throws InstantiationException + * @throws InvalidToken + */ + private void replicate() + throws ServiceFailure, InvalidRequest, IOException, FileUploadException, + MarshallingException, NotImplemented, NotAuthorized, InsufficientResources, + UnsupportedType, InstantiationException, IllegalAccessException, InvalidToken { + + logMetacat.debug("in POST replicate()"); + ReadOnlyChecker checker = new ReadOnlyChecker(); + boolean isReadOnlyMode = checker.isReadOnly(); + if (isReadOnlyMode) { + throw new ServiceFailure("2151", ReadOnlyChecker.DATAONEERROR); + } + + // somewhat unorthodox, but the call is asynchronous and we'd like to return this info sooner + boolean allowed = false; + if (session == null) { + String msg = "No session was provided."; + NotAuthorized failure = new NotAuthorized("2152", msg); + throw failure; + } else { + // TODO: should we refactore replicate() in MNodeservice to not replicate, it would avoid a possible second listNodes call... + D1AuthHelper authDel = new D1AuthHelper(request, null, "2152", "????"); + authDel.doAdminAuthorization(session); // allowed = MNodeService.getInstance(request).isAdminAuthorized(session); // if (!allowed) { // String msg = "User is not an admin user"; // NotAuthorized failure = new NotAuthorized("2152", msg); // throw failure; // } - } - - // parse the systemMetadata - Map files = collectMultipartFiles(); - final SystemMetadata sysmeta = TypeMarshaller.unmarshalTypeFromFile(SystemMetadata.class, files.get("sysmeta")); - - String sn = multipartparams.get("sourceNode").get(0); - logMetacat.debug("sourceNode: " + sn); - final NodeReference sourceNode = new NodeReference(); - sourceNode.setValue(sn); - - // run it in a thread to avoid connection timeout - final String ipAddress = request.getRemoteAddr(); - final String userAgent = request.getHeader("User-Agent"); - Runnable runner = new Runnable() { - @Override - public void run() { - try { - MNodeService.getInstance(request, ipAddress, userAgent).replicate(session, sysmeta, sourceNode); - } catch (Exception e) { - logMetacat.error("Error running replication: " + e.getMessage(), e); - throw new RuntimeException(e.getMessage(), e); - } - } - }; - // submit the task, and that's it - executor.submit(runner); - - // thread was started, so we return success - response.setStatus(200); - - } - - /** - * Handle the getReplica action for the MN - * - * @param id the identifier for the object - * @throws NotFound - * @throws ServiceFailure - * @throws NotImplemented - * @throws NotAuthorized - * @throws InvalidToken - * @throws InvalidRequest - */ - private void getReplica(String id) - throws InvalidRequest, InvalidToken, NotAuthorized, NotImplemented, - ServiceFailure, NotFound { - - Identifier pid = new Identifier(); - pid.setValue(id); - OutputStream out = null; - InputStream dataBytes = null; - - try { - // call the service - dataBytes = MNodeService.getInstance(request).getReplica(session, pid); - - response.setContentType("application/octet-stream"); - response.setStatus(200); - out = response.getOutputStream(); - // write the object to the output stream - IOUtils.copyLarge(dataBytes, out); - IOUtils.closeQuietly(out); - } catch (IOException e) { - String msg = "There was an error writing the output: " + e.getMessage(); - logMetacat.error(msg); - throw new ServiceFailure("2181", msg); - - } - - } - - /** - * Get the Node information - * - * @throws MarshallingException - * @throws IOException - * @throws InvalidRequest - * @throws ServiceFailure - * @throws NotAuthorized - * @throws NotImplemented - */ - private void node() - throws MarshallingException, IOException, NotImplemented, NotAuthorized, ServiceFailure, InvalidRequest { - - Node n = MNodeService.getInstance(request).getCapabilities(); - - response.setContentType("text/xml"); - response.setStatus(200); - TypeMarshaller.marshalTypeToOutputStream(n, response.getOutputStream()); - IOUtils.closeQuietly(response.getOutputStream()); - - } - - /** - * MN_crud.describe() - * http://mule1.dataone.org/ArchitectureDocs/mn_api_crud.html#MN_crud.describe - * - * @param pid - * @throws InvalidRequest - * @throws NotImplemented - * @throws NotFound - * @throws NotAuthorized - * @throws ServiceFailure - * @throws InvalidToken - */ - private void describeObject(String pid) throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented, InvalidRequest { - - response.setContentType("text/xml"); - - Identifier id = new Identifier(); - id.setValue(pid); - - DescribeResponse dr = null; - try { - dr = MNodeService.getInstance(request).describe(session, id); - } catch (BaseException e) { - response.setStatus(e.getCode()); - response.addHeader("DataONE-Exception-Name", e.getClass().getName()); - response.addHeader("DataONE-Exception-DetailCode", e.getDetail_code()); - response.addHeader("DataONE-Exception-Description", e.getDescription()); - response.addHeader("DataONE-Exception-PID", id.getValue()); - return; - } - - response.setStatus(200); - - //response.addHeader("pid", pid); - response.addHeader("DataONE-Checksum", dr.getDataONE_Checksum().getAlgorithm() + "," + dr.getDataONE_Checksum().getValue()); - response.addHeader("Content-Length", dr.getContent_Length() + ""); - response.addHeader("Last-Modified", DateTimeMarshaller.serializeDateToUTC(dr.getLast_Modified())); - response.addHeader("DataONE-ObjectFormat", dr.getDataONE_ObjectFormatIdentifier().getValue()); - response.addHeader("DataONE-SerialVersion", dr.getSerialVersion().toString()); - - - } - - /** - * get the logs based on passed params. Available - * See http://mule1.dataone.org/ArchitectureDocs/mn_api_crud.html#MN_crud.getLogRecords - * for more info - * - * @throws NotImplemented - * @throws InvalidRequest - * @throws NotAuthorized - * @throws ServiceFailure - * @throws InvalidToken - * @throws IOException - * @throws MarshallingException - */ - private void getLog() throws InvalidToken, ServiceFailure, NotAuthorized, InvalidRequest, NotImplemented, IOException, MarshallingException { - - Date fromDate = null; - Date toDate = null; - String event = null; - Integer start = null; - Integer count = null; - String pidFilter = null; - - try { - String fromDateS = params.get("fromDate")[0]; - logMetacat.debug("param fromDateS: " + fromDateS); - fromDate = DateTimeMarshaller.deserializeDateToUTC(fromDateS); - } catch (Exception e) { - logMetacat.warn("Could not parse fromDate: " + e.getMessage()); - } - try { - String toDateS = params.get("toDate")[0]; - logMetacat.debug("param toDateS: " + toDateS); - toDate = DateTimeMarshaller.deserializeDateToUTC(toDateS); - } catch (Exception e) { - logMetacat.warn("Could not parse toDate: " + e.getMessage()); - } - try { - event = params.get("event")[0]; - } catch (Exception e) { - logMetacat.warn("Could not parse event: " + e.getMessage()); - } - logMetacat.debug("fromDate: " + fromDate + " toDate: " + toDate); - - try { - start = Integer.parseInt(params.get("start")[0]); - } catch (Exception e) { - logMetacat.warn("Could not parse start: " + e.getMessage()); - } - try { - count = Integer.parseInt(params.get("count")[0]); - } catch (Exception e) { - logMetacat.warn("Could not parse count: " + e.getMessage()); - } - try { - pidFilter = params.get("idFilter")[0]; - } catch (Exception e) { - logMetacat.warn("Could not parse pidFilter: " + e.getMessage()); - } - - logMetacat.debug("calling getLogRecords"); - Log log = MNodeService.getInstance(request).getLogRecords(session, fromDate, toDate, event, pidFilter, start, count); - - OutputStream out = response.getOutputStream(); - response.setStatus(200); - response.setContentType("text/xml"); - - TypeMarshaller.marshalTypeToOutputStream(log, out); - IOUtils.closeQuietly(out); - } - - /** - * Implements REST version of DataONE CRUD API --> get - * - * @param pid ID of data object to be read - * @throws NotImplemented - * @throws InvalidRequest - * @throws NotFound - * @throws NotAuthorized - * @throws ServiceFailure - * @throws InvalidToken - * @throws IOException - * @throws MarshallingException - */ - protected void getObject(String pid) throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, InvalidRequest, NotImplemented, IOException, MarshallingException { - OutputStream out = null; - - if (pid != null) { //get a specific document - long start = System.currentTimeMillis(); - Identifier id = new Identifier(); - id.setValue(pid); - - SystemMetadata sm = MNodeService.getInstance(request).getSystemMetadata(session, id); - - // set the headers for the content - String mimeType = null; - String charset = null; - ObjectFormat objectFormat = null; - - try { - objectFormat = ObjectFormatCache.getInstance().getFormat(sm.getFormatId()); - } catch (BaseException be) { - logMetacat.warn("Could not lookup ObjectFormat for: " + sm.getFormatId(), be); - } - // do we have mediaType/encoding in SM? - MediaType mediaType = sm.getMediaType(); - if (mediaType == null && objectFormat != null) { - try { - mediaType = objectFormat.getMediaType(); - } catch (Exception e) { - logMetacat.warn("Could not lookup MediaType for: " + sm.getFormatId(), e); - } - } - if (mediaType != null) { - mimeType = mediaType.getName(); - if (mediaType.getPropertyList() != null) { - Iterator iter = mediaType.getPropertyList().iterator(); - while (iter.hasNext()) { - MediaTypeProperty mtp = iter.next(); - if (mtp.getName().equalsIgnoreCase("charset")) { - charset = mtp.getValue(); - mimeType += "; charset=" + charset; - break; - } - } - } - } - // check object format - - // use the fallback from v1 impl - if (mimeType == null) { - mimeType = ObjectFormatInfo.instance().getMimeType(sm.getFormatId().getValue()); - - // still null? - if (mimeType == null) { - mimeType = "application/octet-stream"; - } - } - - // check for filename in SM first - String filename = sm.getFileName(); - // then fallback to using id and extension - if (filename == null) { - String extension = null; - if (objectFormat != null) { - extension = objectFormat.getExtension(); - } - if (extension == null) { - extension = ObjectFormatInfo.instance().getExtension(sm.getFormatId().getValue()); - } - filename = id.getValue(); - if (extension != null && filename != null && !filename.endsWith(extension)) { - filename = id.getValue() + "." + extension; - } - } - response.setContentType(mimeType); - response.setHeader("Content-Disposition", "inline; filename=\"" + filename + "\""); - InputStream data = null; - try { - data = MNodeService.getInstance(request).get(session, id); - out = response.getOutputStream(); - response.setStatus(200); - IOUtils.copyLarge(data, out); - IOUtils.closeQuietly(out); - } finally { - if (data != null) { - IOUtils.closeQuietly(data); - } - } - long end = System.currentTimeMillis(); - logMetacat.info(Settings.PERFORMANCELOG + pid + Settings.PERFORMANCELOG_GET_METHOD + " Total get method" + Settings.PERFORMANCELOG_DURATION + (end - start) / 1000); - } else { //call listObjects with specified params - Date startTime = null; - Date endTime = null; - ObjectFormatIdentifier formatId = null; - Identifier identifier = null; - boolean replicaStatus = true; - int start = 0; - //TODO: make the max count into a const - int count = 1000; - Enumeration paramlist = request.getParameterNames(); - while (paramlist.hasMoreElements()) { //parse the params and make the crud call - String name = (String) paramlist.nextElement(); - String[] value = (String[]) request.getParameterValues(name); - - if (name.equals("fromDate") && value != null) { - try { - //startTime = dateFormat.parse(value[0]); - startTime = DateTimeMarshaller.deserializeDateToUTC(value[0]); - //startTime = parseDateAndConvertToGMT(value[0]); - } catch (Exception e) { //if we can't parse it, just don't use the fromDate param - logMetacat.warn("Could not parse fromDate: " + value[0], e); - throw new InvalidRequest("1540", "Could not parse fromDate: " + value[0] + " since " + e.getMessage()); - //startTime = null; - } - } else if (name.equals("toDate") && value != null) { - try { - endTime = DateTimeMarshaller.deserializeDateToUTC(value[0]); - } catch (Exception e) { //if we can't parse it, just don't use the toDate param - logMetacat.warn("Could not parse toDate: " + value[0], e); - throw new InvalidRequest("1540", "Could not parse toDate: " + value[0] + " since " + e.getMessage()); - //endTime = null; - } - } else if (name.equals("formatId") && value != null) { - formatId = new ObjectFormatIdentifier(); - formatId.setValue(value[0]); - } else if (name.equals("identifier") && value != null) { - identifier = new Identifier(); - identifier.setValue(value[0]); - } else if (name.equals("replicaStatus") && value != null) { - if (value != null && - value.length > 0 && - (value[0].equalsIgnoreCase("false") || value[0].equalsIgnoreCase("no"))) { - replicaStatus = false; - } - } else if (name.equals("start") && value != null) { - start = new Integer(value[0]).intValue(); - } else if (name.equals("count") && value != null) { - count = new Integer(value[0]).intValue(); - } - } - //make the crud call - logMetacat.debug("session: " + session + " startTime: " + startTime + - " endTime: " + endTime + " formatId: " + - formatId + " replicaStatus: " + replicaStatus + - " start: " + start + " count: " + count); - - ObjectList ol = - MNodeService.getInstance(request).listObjects(session, startTime, endTime, - formatId, identifier, replicaStatus, start, count); - - out = response.getOutputStream(); - response.setStatus(200); - response.setContentType("text/xml"); - // Serialize and write it to the output stream - TypeMarshaller.marshalTypeToOutputStream(ol, out); - IOUtils.closeQuietly(out); - } - - } - - - /** - * Retrieve data package as Bagit zip - * - * @param format - * @param pid The pid of the resource map defining the pacakage - * @throws NotImplemented - * @throws NotFound - * @throws NotAuthorized - * @throws ServiceFailure - * @throws InvalidToken - * @throws IOException - * @throws InvalidRequest - */ - protected void getPackage(String format, String pid) throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented, IOException, InvalidRequest { - - long start = System.currentTimeMillis(); - Identifier id = new Identifier(); - id.setValue(pid); - ObjectFormatIdentifier formatId = null; - if (format != null) { - formatId = new ObjectFormatIdentifier(); - formatId.setValue(format); - } - InputStream is = null; - try { - is = MNodeService.getInstance(request).getPackage(session, formatId , id); - //Use the pid as the file name prefix, replacing all non-word characters - String filename = pid.replaceAll("\\W", "_") + ".zip"; - response.setHeader("Content-Disposition", "inline; filename=\"" + filename + "\""); - response.setContentType("application/zip"); - response.setStatus(200); - OutputStream out = response.getOutputStream(); - - // write it to the output stream - IOUtils.copyLarge(is, out); - long end = System.currentTimeMillis(); - logMetacat.info(Settings.PERFORMANCELOG + pid + Settings.PERFORMANCELOG_GET_PACKAGE_METHOD + " Total getPackage method" + Settings.PERFORMANCELOG_DURATION + (end - start) / 1000); - } finally { - IOUtils.closeQuietly(is); - } - } - - protected void publish(String pid) throws InvalidToken, ServiceFailure, - NotAuthorized, NotFound, NotImplemented, IOException, - MarshallingException, InvalidRequest, IdentifierNotUnique, - UnsupportedType, InsufficientResources, InvalidSystemMetadata { - - // publish the object - Identifier originalIdentifier = new Identifier(); - originalIdentifier.setValue(pid); - Identifier newIdentifier = MNodeService.getInstance(request).publish(session, originalIdentifier); - - response.setStatus(200); - response.setContentType("text/xml"); - OutputStream out = response.getOutputStream(); - - // write new identifier to the output stream - TypeMarshaller.marshalTypeToOutputStream(newIdentifier, out); - IOUtils.closeQuietly(out); - } + } + + // parse the systemMetadata + Map files = collectMultipartFiles(); + final SystemMetadata sysmeta = TypeMarshaller.unmarshalTypeFromFile(SystemMetadata.class, files.get("sysmeta")); + + String sn = multipartparams.get("sourceNode").get(0); + logMetacat.debug("sourceNode: " + sn); + final NodeReference sourceNode = new NodeReference(); + sourceNode.setValue(sn); + + // run it in a thread to avoid connection timeout + final String ipAddress = request.getRemoteAddr(); + final String userAgent = request.getHeader("User-Agent"); + Runnable runner = new Runnable() { + @Override + public void run() { + try { + MNodeService.getInstance(request, ipAddress, userAgent).replicate(session, sysmeta, sourceNode); + } catch (Exception e) { + logMetacat.error("Error running replication: " + e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); + } + } + }; + // submit the task, and that's it + executor.submit(runner); + + // thread was started, so we return success + response.setStatus(200); + + } + + /** + * Handle the getReplica action for the MN + * + * @param id the identifier for the object + * @throws NotFound + * @throws ServiceFailure + * @throws NotImplemented + * @throws NotAuthorized + * @throws InvalidToken + * @throws InvalidRequest + */ + private void getReplica(String id) + throws InvalidRequest, InvalidToken, NotAuthorized, NotImplemented, + ServiceFailure, NotFound { + + Identifier pid = new Identifier(); + pid.setValue(id); + OutputStream out = null; + InputStream dataBytes = null; + + try { + // call the service + dataBytes = MNodeService.getInstance(request).getReplica(session, pid); + + response.setContentType("application/octet-stream"); + response.setStatus(200); + out = response.getOutputStream(); + // write the object to the output stream + IOUtils.copyLarge(dataBytes, out); + IOUtils.closeQuietly(out); + } catch (IOException e) { + String msg = "There was an error writing the output: " + e.getMessage(); + logMetacat.error(msg); + throw new ServiceFailure("2181", msg); + + } + + } + + /** + * Get the Node information + * + * @throws MarshallingException + * @throws IOException + * @throws InvalidRequest + * @throws ServiceFailure + * @throws NotAuthorized + * @throws NotImplemented + */ + private void node() + throws MarshallingException, IOException, NotImplemented, NotAuthorized, ServiceFailure, InvalidRequest { + + Node n = MNodeService.getInstance(request).getCapabilities(); + + response.setContentType("text/xml"); + response.setStatus(200); + TypeMarshaller.marshalTypeToOutputStream(n, response.getOutputStream()); + IOUtils.closeQuietly(response.getOutputStream()); + + } + + /** + * MN_crud.describe() + * http://mule1.dataone.org/ArchitectureDocs/mn_api_crud.html#MN_crud.describe + * + * @param pid + * @throws InvalidRequest + * @throws NotImplemented + * @throws NotFound + * @throws NotAuthorized + * @throws ServiceFailure + * @throws InvalidToken + */ + private void describeObject(String pid) throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented, InvalidRequest { + + response.setContentType("text/xml"); + + Identifier id = new Identifier(); + id.setValue(pid); + + DescribeResponse dr = null; + try { + dr = MNodeService.getInstance(request).describe(session, id); + } catch (BaseException e) { + response.setStatus(e.getCode()); + response.addHeader("DataONE-Exception-Name", e.getClass().getName()); + response.addHeader("DataONE-Exception-DetailCode", e.getDetail_code()); + response.addHeader("DataONE-Exception-Description", e.getDescription()); + response.addHeader("DataONE-Exception-PID", id.getValue()); + return; + } + + response.setStatus(200); + + //response.addHeader("pid", pid); + response.addHeader("DataONE-Checksum", dr.getDataONE_Checksum().getAlgorithm() + "," + dr.getDataONE_Checksum().getValue()); + response.addHeader("Content-Length", dr.getContent_Length() + ""); + response.addHeader("Last-Modified", DateTimeMarshaller.serializeDateToUTC(dr.getLast_Modified())); + response.addHeader("DataONE-ObjectFormat", dr.getDataONE_ObjectFormatIdentifier().getValue()); + response.addHeader("DataONE-SerialVersion", dr.getSerialVersion().toString()); + + + } + + /** + * get the logs based on passed params. Available + * See http://mule1.dataone.org/ArchitectureDocs/mn_api_crud.html#MN_crud.getLogRecords + * for more info + * + * @throws NotImplemented + * @throws InvalidRequest + * @throws NotAuthorized + * @throws ServiceFailure + * @throws InvalidToken + * @throws IOException + * @throws MarshallingException + */ + private void getLog() throws InvalidToken, ServiceFailure, NotAuthorized, InvalidRequest, NotImplemented, IOException, MarshallingException { + + Date fromDate = null; + Date toDate = null; + String event = null; + Integer start = null; + Integer count = null; + String pidFilter = null; + + try { + String fromDateS = params.get("fromDate")[0]; + logMetacat.debug("param fromDateS: " + fromDateS); + fromDate = DateTimeMarshaller.deserializeDateToUTC(fromDateS); + } catch (Exception e) { + logMetacat.warn("Could not parse fromDate: " + e.getMessage()); + } + try { + String toDateS = params.get("toDate")[0]; + logMetacat.debug("param toDateS: " + toDateS); + toDate = DateTimeMarshaller.deserializeDateToUTC(toDateS); + } catch (Exception e) { + logMetacat.warn("Could not parse toDate: " + e.getMessage()); + } + try { + event = params.get("event")[0]; + } catch (Exception e) { + logMetacat.warn("Could not parse event: " + e.getMessage()); + } + logMetacat.debug("fromDate: " + fromDate + " toDate: " + toDate); + + try { + start = Integer.parseInt(params.get("start")[0]); + } catch (Exception e) { + logMetacat.warn("Could not parse start: " + e.getMessage()); + } + try { + count = Integer.parseInt(params.get("count")[0]); + } catch (Exception e) { + logMetacat.warn("Could not parse count: " + e.getMessage()); + } + try { + pidFilter = params.get("idFilter")[0]; + } catch (Exception e) { + logMetacat.warn("Could not parse pidFilter: " + e.getMessage()); + } + + logMetacat.debug("calling getLogRecords"); + Log log = MNodeService.getInstance(request).getLogRecords(session, fromDate, toDate, event, pidFilter, start, count); + + OutputStream out = response.getOutputStream(); + response.setStatus(200); + response.setContentType("text/xml"); + + TypeMarshaller.marshalTypeToOutputStream(log, out); + IOUtils.closeQuietly(out); + } + + /** + * Implements REST version of DataONE CRUD API --> get + * + * @param pid ID of data object to be read + * @throws NotImplemented + * @throws InvalidRequest + * @throws NotFound + * @throws NotAuthorized + * @throws ServiceFailure + * @throws InvalidToken + * @throws IOException + * @throws MarshallingException + */ + protected void getObject(String pid) throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, InvalidRequest, NotImplemented, IOException, MarshallingException { + OutputStream out = null; + + if (pid != null) { //get a specific document + long start = System.currentTimeMillis(); + Identifier id = new Identifier(); + id.setValue(pid); + + SystemMetadata sm = MNodeService.getInstance(request).getSystemMetadata(session, id); + + // set the headers for the content + String mimeType = null; + String charset = null; + ObjectFormat objectFormat = null; + + try { + objectFormat = ObjectFormatCache.getInstance().getFormat(sm.getFormatId()); + } catch (BaseException be) { + logMetacat.warn("Could not lookup ObjectFormat for: " + sm.getFormatId(), be); + } + // do we have mediaType/encoding in SM? + MediaType mediaType = sm.getMediaType(); + if (mediaType == null && objectFormat != null) { + try { + mediaType = objectFormat.getMediaType(); + } catch (Exception e) { + logMetacat.warn("Could not lookup MediaType for: " + sm.getFormatId(), e); + } + } + if (mediaType != null) { + mimeType = mediaType.getName(); + if (mediaType.getPropertyList() != null) { + Iterator iter = mediaType.getPropertyList().iterator(); + while (iter.hasNext()) { + MediaTypeProperty mtp = iter.next(); + if (mtp.getName().equalsIgnoreCase("charset")) { + charset = mtp.getValue(); + mimeType += "; charset=" + charset; + break; + } + } + } + } + // check object format + + // use the fallback from v1 impl + if (mimeType == null) { + mimeType = ObjectFormatInfo.instance().getMimeType(sm.getFormatId().getValue()); + + // still null? + if (mimeType == null) { + mimeType = "application/octet-stream"; + } + } + + // check for filename in SM first + String filename = sm.getFileName(); + // then fallback to using id and extension + if (filename == null) { + String extension = null; + if (objectFormat != null) { + extension = objectFormat.getExtension(); + } + if (extension == null) { + extension = ObjectFormatInfo.instance().getExtension(sm.getFormatId().getValue()); + } + filename = id.getValue(); + if (extension != null && filename != null && !filename.endsWith(extension)) { + filename = id.getValue() + "." + extension; + } + } + response.setContentType(mimeType); + response.setHeader("Content-Disposition", "inline; filename=\"" + filename + "\""); + InputStream data = null; + try { + data = MNodeService.getInstance(request).get(session, id); + out = response.getOutputStream(); + response.setStatus(200); + IOUtils.copyLarge(data, out); + IOUtils.closeQuietly(out); + } finally { + if (data != null) { + IOUtils.closeQuietly(data); + } + } + long end = System.currentTimeMillis(); + logMetacat.info(Settings.PERFORMANCELOG + pid + Settings.PERFORMANCELOG_GET_METHOD + " Total get method" + Settings.PERFORMANCELOG_DURATION + (end - start) / 1000); + } else { //call listObjects with specified params + Date startTime = null; + Date endTime = null; + ObjectFormatIdentifier formatId = null; + Identifier identifier = null; + boolean replicaStatus = true; + int start = 0; + //TODO: make the max count into a const + int count = 1000; + Enumeration paramlist = request.getParameterNames(); + while (paramlist.hasMoreElements()) { //parse the params and make the crud call + String name = (String) paramlist.nextElement(); + String[] value = (String[]) request.getParameterValues(name); + + if (name.equals("fromDate") && value != null) { + try { + //startTime = dateFormat.parse(value[0]); + startTime = DateTimeMarshaller.deserializeDateToUTC(value[0]); + //startTime = parseDateAndConvertToGMT(value[0]); + } catch (Exception e) { //if we can't parse it, just don't use the fromDate param + logMetacat.warn("Could not parse fromDate: " + value[0], e); + throw new InvalidRequest("1540", "Could not parse fromDate: " + value[0] + " since " + e.getMessage()); + //startTime = null; + } + } else if (name.equals("toDate") && value != null) { + try { + endTime = DateTimeMarshaller.deserializeDateToUTC(value[0]); + } catch (Exception e) { //if we can't parse it, just don't use the toDate param + logMetacat.warn("Could not parse toDate: " + value[0], e); + throw new InvalidRequest("1540", "Could not parse toDate: " + value[0] + " since " + e.getMessage()); + //endTime = null; + } + } else if (name.equals("formatId") && value != null) { + formatId = new ObjectFormatIdentifier(); + formatId.setValue(value[0]); + } else if (name.equals("identifier") && value != null) { + identifier = new Identifier(); + identifier.setValue(value[0]); + } else if (name.equals("replicaStatus") && value != null) { + if (value != null && + value.length > 0 && + (value[0].equalsIgnoreCase("false") || value[0].equalsIgnoreCase("no"))) { + replicaStatus = false; + } + } else if (name.equals("start") && value != null) { + start = new Integer(value[0]).intValue(); + } else if (name.equals("count") && value != null) { + count = new Integer(value[0]).intValue(); + } + } + //make the crud call + logMetacat.debug("session: " + session + " startTime: " + startTime + + " endTime: " + endTime + " formatId: " + + formatId + " replicaStatus: " + replicaStatus + + " start: " + start + " count: " + count); + + ObjectList ol = + MNodeService.getInstance(request).listObjects(session, startTime, endTime, + formatId, identifier, replicaStatus, start, count); + + out = response.getOutputStream(); + response.setStatus(200); + response.setContentType("text/xml"); + // Serialize and write it to the output stream + TypeMarshaller.marshalTypeToOutputStream(ol, out); + IOUtils.closeQuietly(out); + } + + } + + + /** + * Retrieve data package as Bagit zip + * + * @param format + * @param pid The pid of the resource map defining the pacakage + * @throws NotImplemented + * @throws NotFound + * @throws NotAuthorized + * @throws ServiceFailure + * @throws InvalidToken + * @throws IOException + * @throws InvalidRequest + */ + protected void getPackage(String format, String pid) throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented, IOException, InvalidRequest { + + long start = System.currentTimeMillis(); + Identifier id = new Identifier(); + id.setValue(pid); + ObjectFormatIdentifier formatId = null; + if (format != null) { + formatId = new ObjectFormatIdentifier(); + formatId.setValue(format); + } + InputStream is = null; + try { + is = MNodeService.getInstance(request).getPackage(session, formatId , id); + //Use the pid as the file name prefix, replacing all non-word characters + String filename = pid.replaceAll("\\W", "_") + ".zip"; + response.setHeader("Content-Disposition", "inline; filename=\"" + filename + "\""); + response.setContentType("application/zip"); + response.setStatus(200); + OutputStream out = response.getOutputStream(); + + // write it to the output stream + IOUtils.copyLarge(is, out); + long end = System.currentTimeMillis(); + logMetacat.info(Settings.PERFORMANCELOG + pid + Settings.PERFORMANCELOG_GET_PACKAGE_METHOD + " Total getPackage method" + Settings.PERFORMANCELOG_DURATION + (end - start) / 1000); + } finally { + IOUtils.closeQuietly(is); + } + } + + protected void publish(String pid) throws InvalidToken, ServiceFailure, + NotAuthorized, NotFound, NotImplemented, IOException, + MarshallingException, InvalidRequest, IdentifierNotUnique, + UnsupportedType, InsufficientResources, InvalidSystemMetadata { + + // publish the object + Identifier originalIdentifier = new Identifier(); + originalIdentifier.setValue(pid); + Identifier newIdentifier = MNodeService.getInstance(request).publish(session, originalIdentifier); + + response.setStatus(200); + response.setContentType("text/xml"); + OutputStream out = response.getOutputStream(); + + // write new identifier to the output stream + TypeMarshaller.marshalTypeToOutputStream(newIdentifier, out); + IOUtils.closeQuietly(out); + } /** * Retrieve System Metadata @@ -1732,63 +1732,63 @@ private void archive(String pid) throws InvalidToken, ServiceFailure, NotAuthori logMetacat.info(Settings.PERFORMANCELOG + pid + Settings.PERFORMANCELOG_ARCHIVE_METHOD + " Total archive method" + Settings.PERFORMANCELOG_DURATION + (end-start)/1000); } - protected SynchronizationFailed collectSynchronizationFailed() throws IOException, ServiceFailure, InvalidRequest, MarshallingException, InstantiationException, IllegalAccessException, ParserConfigurationException, SAXException { - - // Read the incoming data from its Mime Multipart encoding - logMetacat.debug("Disassembling MIME multipart form"); - InputStream sf = null; - - // handle MMP inputs - File tmpDir = getTempDirectory(); - logMetacat.debug("temp dir: " + tmpDir.getAbsolutePath()); - MultipartRequestResolver mrr = - new MultipartRequestResolver(tmpDir.getAbsolutePath(), MAX_UPLOAD_SIZE, 0); - MultipartRequest mr = null; - try { - mr = mrr.resolveMultipart(request); - } catch (Exception e) { - throw new ServiceFailure("2161", - "Could not resolve multipart: " + e.getMessage()); - } - logMetacat.debug("resolved multipart request"); - Map files = mr.getMultipartFiles(); - if (files == null || files.keySet() == null) { - throw new InvalidRequest("2163", - "must have multipart file with name 'message'"); - } - logMetacat.debug("got multipart files"); - - multipartparams = mr.getMultipartParameters(); - - File sfFile = files.get("message"); - if (sfFile == null) { - throw new InvalidRequest("2163", - "Missing the required file-part 'message' from the multipart request."); - } - logMetacat.debug("sfFile: " + sfFile.getAbsolutePath()); - sf = new FileInputStream(sfFile); - - SynchronizationFailed syncFailed = (SynchronizationFailed) ExceptionHandler.deserializeXml(sf, "Error deserializing exception"); - return syncFailed; - } - - /** - * Update the system metadata for a specified identifier - * @throws ServiceFailure - * @throws InvalidRequest - * @throws InstantiationException - * @throws IllegalAccessException - * @throws IOException - * @throws MarshallingException - * @throws NotImplemented - * @throws NotAuthorized - * @throws InvalidSystemMetadata - * @throws InvalidToken - */ - protected void updateSystemMetadata() throws ServiceFailure, InvalidRequest, - InstantiationException, IllegalAccessException, IOException, MarshallingException, NotImplemented, - NotAuthorized, InvalidSystemMetadata, InvalidToken { - // Read the incoming data from its Mime Multipart encoding + protected SynchronizationFailed collectSynchronizationFailed() throws IOException, ServiceFailure, InvalidRequest, MarshallingException, InstantiationException, IllegalAccessException, ParserConfigurationException, SAXException { + + // Read the incoming data from its Mime Multipart encoding + logMetacat.debug("Disassembling MIME multipart form"); + InputStream sf = null; + + // handle MMP inputs + File tmpDir = getTempDirectory(); + logMetacat.debug("temp dir: " + tmpDir.getAbsolutePath()); + MultipartRequestResolver mrr = + new MultipartRequestResolver(tmpDir.getAbsolutePath(), MAX_UPLOAD_SIZE, 0); + MultipartRequest mr = null; + try { + mr = mrr.resolveMultipart(request); + } catch (Exception e) { + throw new ServiceFailure("2161", + "Could not resolve multipart: " + e.getMessage()); + } + logMetacat.debug("resolved multipart request"); + Map files = mr.getMultipartFiles(); + if (files == null || files.keySet() == null) { + throw new InvalidRequest("2163", + "must have multipart file with name 'message'"); + } + logMetacat.debug("got multipart files"); + + multipartparams = mr.getMultipartParameters(); + + File sfFile = files.get("message"); + if (sfFile == null) { + throw new InvalidRequest("2163", + "Missing the required file-part 'message' from the multipart request."); + } + logMetacat.debug("sfFile: " + sfFile.getAbsolutePath()); + sf = new FileInputStream(sfFile); + + SynchronizationFailed syncFailed = (SynchronizationFailed) ExceptionHandler.deserializeXml(sf, "Error deserializing exception"); + return syncFailed; + } + + /** + * Update the system metadata for a specified identifier + * @throws ServiceFailure + * @throws InvalidRequest + * @throws InstantiationException + * @throws IllegalAccessException + * @throws IOException + * @throws MarshallingException + * @throws NotImplemented + * @throws NotAuthorized + * @throws InvalidSystemMetadata + * @throws InvalidToken + */ + protected void updateSystemMetadata() throws ServiceFailure, InvalidRequest, + InstantiationException, IllegalAccessException, IOException, MarshallingException, NotImplemented, + NotAuthorized, InvalidSystemMetadata, InvalidToken { + // Read the incoming data from its Mime Multipart encoding Map files = collectMultipartFiles(); // get the encoded pid string from the body and make the object @@ -1806,6 +1806,6 @@ protected void updateSystemMetadata() throws ServiceFailure, InvalidRequest, logMetacat.debug("updating system metadata with pid " + pid.getValue()); MNodeService.getInstance(request).updateSystemMetadata(session, pid, systemMetadata); - } + } } From 0b30f545799b3c96e2d4f3d25535cfd619671be8 Mon Sep 17 00:00:00 2001 From: Thomas Thelen Date: Wed, 21 Jul 2021 19:27:40 -0700 Subject: [PATCH 19/45] Revert v2/MNResourceHandler.java --- .../restservice/v2/MNResourceHandler.java | 1593 +++++++++-------- 1 file changed, 807 insertions(+), 786 deletions(-) diff --git a/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java b/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java index bacf3f973..cf3431132 100644 --- a/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java +++ b/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java @@ -36,7 +36,6 @@ import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.zip.ZipOutputStream; import javax.servlet.ServletContext; import javax.servlet.http.HttpServletRequest; @@ -87,7 +86,6 @@ import org.dataone.service.util.DateTimeMarshaller; import org.dataone.service.util.ExceptionHandler; import org.dataone.service.util.TypeMarshaller; -import org.dataone.speedbagit.SpeedBagIt; import org.xml.sax.SAXException; import edu.ucsb.nceas.metacat.MetaCatServlet; @@ -102,6 +100,7 @@ import edu.ucsb.nceas.metacat.restservice.multipart.DetailedFileInputStream; import edu.ucsb.nceas.metacat.restservice.multipart.MultipartRequestWithSysmeta; import edu.ucsb.nceas.metacat.restservice.multipart.StreamingMultipartRequestResolver; +import edu.ucsb.nceas.metacat.util.DeleteOnCloseFileInputStream; import edu.ucsb.nceas.utilities.PropertyNotFoundException; /** @@ -134,7 +133,6 @@ * delete() - DELETE /d1/mn/object/PID * archive() - PUT /d1/mn/archive/PID * updateSystemMetadata() - PUT /d1/mn/meta - * systemMetadataChanged() - POST /dirtySystemMetadata/PID * * MNReplication @@ -161,75 +159,77 @@ public class MNResourceHandler extends D1ResourceHandler { protected static final String RESOURCE_WHOAMI = "whoami"; + + // shared executor - private static ExecutorService executor = null; + private static ExecutorService executor = null; - static { - // use a shared executor service with nThreads == one less than available processors - int availableProcessors = Runtime.getRuntime().availableProcessors(); + static { + // use a shared executor service with nThreads == one less than available processors + int availableProcessors = Runtime.getRuntime().availableProcessors(); int nThreads = availableProcessors * 1; nThreads--; nThreads = Math.max(1, nThreads); - executor = Executors.newFixedThreadPool(nThreads); - } - + executor = Executors.newFixedThreadPool(nThreads); + } + /** * Initializes new instance by setting servlet context,request and response - */ + * */ public MNResourceHandler(ServletContext servletContext, - HttpServletRequest request, HttpServletResponse response) { - super(servletContext, request, response); + HttpServletRequest request, HttpServletResponse response) { + super(servletContext, request, response); logMetacat = LogFactory.getLog(MNResourceHandler.class); } - + @Override protected boolean isD1Enabled() { - - boolean enabled = false; - try { - enabled = Boolean.parseBoolean(PropertyService.getProperty("dataone.mn.services.enabled")); - } catch (PropertyNotFoundException e) { - logMetacat.error("Could not check if DataONE is enabled: " + e.getMessage()); - } - - return enabled; + + boolean enabled = false; + try { + enabled = Boolean.parseBoolean(PropertyService.getProperty("dataone.mn.services.enabled")); + } catch (PropertyNotFoundException e) { + logMetacat.error("Could not check if DataONE is enabled: " + e.getMessage()); + } + + return enabled; } /** - * This function is called from REST API servlet and handles each request to the servlet - * + * This function is called from REST API servlet and handles each request to the servlet + * * @param httpVerb (GET, POST, PUT or DELETE) */ @Override public void handle(byte httpVerb) { - // prepare the handler - super.handle(httpVerb); - + // prepare the handler + super.handle(httpVerb); + try { - - // only service requests if we have D1 configured - if (!isD1Enabled()) { - ServiceFailure se = new ServiceFailure("0000", "DataONE services are not enabled on this node"); + + // only service requests if we have D1 configured + if (!isD1Enabled()) { + ServiceFailure se = new ServiceFailure("0000", "DataONE services are not enabled on this node"); serializeException(se, response.getOutputStream()); return; - } - - // get the resource + } + + // get the resource String resource = request.getPathInfo(); resource = resource.substring(resource.indexOf("/") + 1); - + // default to node info if (resource.equals("")) { resource = RESOURCE_NODE; } - + // get the rest of the path info String extra = null; - + logMetacat.info("MNResourceHandler.handle - V2 handling verb " + httpVerb + " request with resource '" + resource + "'"); logMetacat.debug("resource: '" + resource + "'"); boolean status = false; - + if (resource != null) { if (resource.startsWith(RESOURCE_NODE)) { @@ -240,36 +240,36 @@ public void handle(byte httpVerb) { logMetacat.debug("Using resource 'token'"); // get if (httpVerb == GET) { - // after the command + // after the command getToken(); status = true; } - + } else if (resource.startsWith(RESOURCE_WHOAMI)) { logMetacat.debug("Using resource 'whoami'"); // get if (httpVerb == GET) { - // after the command + // after the command whoami(); status = true; } - + } else if (resource.startsWith(RESOURCE_IS_AUTHORIZED)) { if (httpVerb == GET) { - // after the command + // after the command extra = parseTrailing(resource, RESOURCE_IS_AUTHORIZED); extra = decode(extra); - // check the access rules - isAuthorized(extra); - status = true; - logMetacat.debug("done getting access"); + // check the access rules + isAuthorized(extra); + status = true; + logMetacat.debug("done getting access"); } } else if (resource.startsWith(RESOURCE_META)) { logMetacat.debug("Using resource 'meta'"); // get if (httpVerb == GET) { logMetacat.debug("Using resource 'meta' for GET"); - // after the command + // after the command extra = parseTrailing(resource, RESOURCE_META); extra = decode(extra); getSystemMetadataObject(extra); @@ -279,7 +279,7 @@ public void handle(byte httpVerb) { updateSystemMetadata(); status = true; } - + } else if (resource.startsWith(RESOURCE_OBJECTS)) { logMetacat.debug("Using resource 'object'"); // after the command @@ -292,7 +292,7 @@ public void handle(byte httpVerb) { getObject(extra); status = true; } else if (httpVerb == POST) { - // part of the params, not the URL + // part of the params, not the URL putObject(null, FUNCTION_NAME_INSERT); status = true; } else if (httpVerb == PUT) { @@ -305,7 +305,7 @@ public void handle(byte httpVerb) { describeObject(extra); status = true; } - + } else if (resource.startsWith(RESOURCE_LOG)) { logMetacat.debug("Using resource 'log'"); // handle log events @@ -326,7 +326,7 @@ public void handle(byte httpVerb) { logMetacat.debug("Using resource 'checksum'"); // handle checksum requests if (httpVerb == GET) { - // after the command + // after the command extra = parseTrailing(resource, Constants.RESOURCE_CHECKSUM); extra = decode(extra); checksum(extra); @@ -335,35 +335,35 @@ public void handle(byte httpVerb) { } else if (resource.startsWith(RESOURCE_MONITOR)) { // there are various parts to monitoring if (httpVerb == GET) { - // after the command + // after the command extra = parseTrailing(resource, RESOURCE_MONITOR); extra = decode(extra); // ping if (extra.toLowerCase().equals("ping")) { logMetacat.debug("processing ping request"); Date result = MNodeService.getInstance(request).ping(); - // TODO: send to output + // TODO: send to output status = true; - + } else if (extra.toLowerCase().equals("status")) { logMetacat.debug("processing status request"); getStatus(); status = true; } - + } } else if (resource.startsWith(RESOURCE_REPLICATE)) { - if (httpVerb == POST) { - logMetacat.debug("processing replicate request"); - replicate(); - status = true; - } + if (httpVerb == POST) { + logMetacat.debug("processing replicate request"); + replicate(); + status = true; + } } else if (resource.startsWith(RESOURCE_ERROR)) { - // sync error - if (httpVerb == POST) { - syncError(); - status = true; - } + // sync error + if (httpVerb == POST) { + syncError(); + status = true; + } } else if (resource.startsWith(RESOURCE_META_CHANGED)) { // system metadata changed if (httpVerb == POST) { @@ -379,53 +379,53 @@ public void handle(byte httpVerb) { status = true; } } else if (resource.startsWith(RESOURCE_QUERY)) { - logMetacat.debug("Using resource " + RESOURCE_QUERY); - // after the command - extra = parseTrailing(resource, RESOURCE_QUERY); - logMetacat.debug("query extra: " + extra); - - String engine = null; - String query = null; - - if (extra != null) { - // get the engine - int engineIndex = extra.length(); - if (extra.indexOf("/") > -1) { - engineIndex = extra.indexOf("/"); - } - engine = extra.substring(0, engineIndex); - engine = decode(engine); - logMetacat.debug("query engine: " + engine); - - // check the query string first - query = request.getQueryString(); - - // if null, look at the whole endpoint - if (query == null) { - // get the query if it is there - query = extra.substring(engineIndex, extra.length()); - if (query != null && query.length() == 0) { - query = null; - } else { - if (query.startsWith("/")) { - query = query.substring(1); - } - } - } - query = decode(query); - logMetacat.debug("query: " + query); - - } - logMetacat.debug("verb:" + httpVerb); - if (httpVerb == GET) { - doQuery(engine, query); - status = true; - } else if (httpVerb == POST) { - doPostQuery(engine); + logMetacat.debug("Using resource " + RESOURCE_QUERY); + // after the command + extra = parseTrailing(resource, RESOURCE_QUERY); + logMetacat.debug("query extra: " + extra); + + String engine = null; + String query = null; + + if (extra != null) { + // get the engine + int engineIndex = extra.length(); + if (extra.indexOf("/") > -1) { + engineIndex = extra.indexOf("/"); + } + engine = extra.substring(0, engineIndex); + engine = decode(engine); + logMetacat.debug("query engine: " + engine); + + // check the query string first + query = request.getQueryString(); + + // if null, look at the whole endpoint + if (query == null) { + // get the query if it is there + query = extra.substring(engineIndex, extra.length()); + if (query != null && query.length() == 0) { + query = null; + } else { + if (query.startsWith("/")) { + query = query.substring(1); + } + } + } + query = decode(query); + logMetacat.debug("query: " + query); + + } + logMetacat.debug("verb:" + httpVerb); + if (httpVerb == GET) { + doQuery(engine, query); + status = true; + } else if (httpVerb == POST) { + doPostQuery(engine); status = true; - } + } } else if (resource.startsWith(RESOURCE_GENERATE_ID)) { - // generate an id + // generate an id if (httpVerb == POST) { generateIdentifier(); status = true; @@ -434,211 +434,211 @@ public void handle(byte httpVerb) { logMetacat.debug("Using resource: " + RESOURCE_PUBLISH); // PUT if (httpVerb == PUT) { - // after the command + // after the command extra = parseTrailing(resource, RESOURCE_PUBLISH); extra = decode(extra); publish(extra); status = true; - } + } } else if (resource.startsWith(RESOURCE_PACKAGE)) { logMetacat.debug("Using resource: " + RESOURCE_PACKAGE); // after the command extra = parseTrailing(resource, RESOURCE_PACKAGE); - + String format = null; - String pid = null; - - if (extra != null) { - // get the format - int formatIndex = extra.length(); - if (extra.indexOf("/") > -1) { - formatIndex = extra.indexOf("/"); - } - format = extra.substring(0, formatIndex); - format = decode(format); - logMetacat.debug("package format: " + format); - - // get the pid if it is there - pid = extra.substring(formatIndex, extra.length()); - if (pid != null && pid.length() == 0) { - pid = null; - } else { - if (pid.startsWith("/")) { - pid = pid.substring(1); - } - } - pid = decode(pid); - logMetacat.debug("pid: " + pid); - - } - + String pid = null; + + if (extra != null) { + // get the format + int formatIndex = extra.length(); + if (extra.indexOf("/") > -1) { + formatIndex = extra.indexOf("/"); + } + format = extra.substring(0, formatIndex); + format = decode(format); + logMetacat.debug("package format: " + format); + + // get the pid if it is there + pid = extra.substring(formatIndex, extra.length()); + if (pid != null && pid.length() == 0) { + pid = null; + } else { + if (pid.startsWith("/")) { + pid = pid.substring(1); + } + } + pid = decode(pid); + logMetacat.debug("pid: " + pid); + + } + // get if (httpVerb == GET) { - + getPackage(format, pid); status = true; - } + } } else if (resource.startsWith(RESOURCE_VIEWS)) { - logMetacat.debug("Using resource " + RESOURCE_VIEWS); - // after the command - extra = parseTrailing(resource, RESOURCE_VIEWS); - logMetacat.debug("view extra: " + extra); - - String format = null; - String pid = null; - - if (extra != null) { - // get the format - int formatIndex = extra.length(); - if (extra.indexOf("/") > -1) { - formatIndex = extra.indexOf("/"); - } - format = extra.substring(0, formatIndex); - format = decode(format); - logMetacat.debug("view format: " + format); - - // get the pid if it is there - pid = extra.substring(formatIndex, extra.length()); - if (pid != null && pid.length() == 0) { - pid = null; - } else { - if (pid.startsWith("/")) { - pid = pid.substring(1); - } - } - pid = decode(pid); - logMetacat.debug("pid: " + pid); - - } - logMetacat.debug("verb:" + httpVerb); - if (httpVerb == GET) { - doViews(format, pid); - status = true; - } + logMetacat.debug("Using resource " + RESOURCE_VIEWS); + // after the command + extra = parseTrailing(resource, RESOURCE_VIEWS); + logMetacat.debug("view extra: " + extra); + + String format = null; + String pid = null; + + if (extra != null) { + // get the format + int formatIndex = extra.length(); + if (extra.indexOf("/") > -1) { + formatIndex = extra.indexOf("/"); + } + format = extra.substring(0, formatIndex); + format = decode(format); + logMetacat.debug("view format: " + format); + + // get the pid if it is there + pid = extra.substring(formatIndex, extra.length()); + if (pid != null && pid.length() == 0) { + pid = null; + } else { + if (pid.startsWith("/")) { + pid = pid.substring(1); + } + } + pid = decode(pid); + logMetacat.debug("pid: " + pid); + + } + logMetacat.debug("verb:" + httpVerb); + if (httpVerb == GET) { + doViews(format, pid); + status = true; + } } else { throw new InvalidRequest("0000", "No resource matched for " + resource); } - + if (!status) { - throw new ServiceFailure("0000", "Unknown error, status = " + status); + throw new ServiceFailure("0000", "Unknown error, status = " + status); } } else { - throw new InvalidRequest("0000", "No resource matched for " + resource); + throw new InvalidRequest("0000", "No resource matched for " + resource); } } catch (BaseException be) { - // report Exceptions as clearly as possible - OutputStream out = null; - try { - out = response.getOutputStream(); - } catch (IOException e) { - logMetacat.error("Could not get output stream from response", e); - } + // report Exceptions as clearly as possible + OutputStream out = null; + try { + out = response.getOutputStream(); + } catch (IOException e) { + logMetacat.error("Could not get output stream from response", e); + } serializeException(be, out); } catch (Exception e) { // report Exceptions as clearly and generically as possible logMetacat.error(e.getClass() + ": " + e.getMessage(), e); - OutputStream out = null; - try { - out = response.getOutputStream(); - } catch (IOException ioe) { - logMetacat.error("Could not get output stream from response", ioe); - } - ServiceFailure se = new ServiceFailure("0000", e.getMessage()); + OutputStream out = null; + try { + out = response.getOutputStream(); + } catch (IOException ioe) { + logMetacat.error("Could not get output stream from response", ioe); + } + ServiceFailure se = new ServiceFailure("0000", e.getMessage()); serializeException(se, out); } } - + private void doQuery(String engine, String query) { - - OutputStream out = null; - - try { - // NOTE: we set the session explicitly for the MNode instance since these methods do not provide a parameter - if (engine == null) { - // just looking for list of engines - MNodeService mnode = MNodeService.getInstance(request); - mnode.setSession(session); - QueryEngineList qel = mnode.listQueryEngines(session); - response.setContentType("text/xml"); - response.setStatus(200); - out = response.getOutputStream(); - TypeMarshaller.marshalTypeToOutputStream(qel, out); - IOUtils.closeQuietly(out); - return; - } else { - if (query != null) { - long start = System.currentTimeMillis(); - MNodeService mnode = MNodeService.getInstance(request); - mnode.setSession(session); - InputStream stream = mnode.query(session, engine, query); - - // set the content-type if we have it from the implementation - if (stream instanceof ContentTypeInputStream) { - //response.setContentType("application/octet-stream"); - //response.setContentType("text/xml"); - response.setContentType(((ContentTypeInputStream) stream).getContentType()); - } - response.setStatus(200); - out = response.getOutputStream(); - // write the results to the output stream - IOUtils.copyLarge(stream, out); - long end = System.currentTimeMillis(); - logMetacat.info(Settings.PERFORMANCELOG + Settings.PERFORMANCELOG_QUERY_METHOD + query + " Total query method" + Settings.PERFORMANCELOG_DURATION + (end - start) / 1000); - IOUtils.closeQuietly(out); - return; - } else { - MNodeService mnode = MNodeService.getInstance(request); - mnode.setSession(session); - QueryEngineDescription qed = mnode.getQueryEngineDescription(session, engine); - response.setContentType("text/xml"); - response.setStatus(200); - out = response.getOutputStream(); - TypeMarshaller.marshalTypeToOutputStream(qed, out); - IOUtils.closeQuietly(out); - return; - } - } - - - } catch (BaseException be) { - // report Exceptions as clearly as possible - try { - out = response.getOutputStream(); - } catch (IOException e) { - logMetacat.error("Could not get output stream from response", e); - } + + OutputStream out = null; + + try { + // NOTE: we set the session explicitly for the MNode instance since these methods do not provide a parameter + if (engine == null) { + // just looking for list of engines + MNodeService mnode = MNodeService.getInstance(request); + mnode.setSession(session); + QueryEngineList qel = mnode.listQueryEngines(session); + response.setContentType("text/xml"); + response.setStatus(200); + out = response.getOutputStream(); + TypeMarshaller.marshalTypeToOutputStream(qel, out); + IOUtils.closeQuietly(out); + return; + } else { + if (query != null) { + long start = System.currentTimeMillis(); + MNodeService mnode = MNodeService.getInstance(request); + mnode.setSession(session); + InputStream stream = mnode.query(session, engine, query); + + // set the content-type if we have it from the implementation + if (stream instanceof ContentTypeInputStream) { + //response.setContentType("application/octet-stream"); + //response.setContentType("text/xml"); + response.setContentType(((ContentTypeInputStream) stream).getContentType()); + } + response.setStatus(200); + out = response.getOutputStream(); + // write the results to the output stream + IOUtils.copyLarge(stream, out); + long end = System.currentTimeMillis(); + logMetacat.info(Settings.PERFORMANCELOG + Settings.PERFORMANCELOG_QUERY_METHOD + query + " Total query method" + Settings.PERFORMANCELOG_DURATION + (end-start)/1000); + IOUtils.closeQuietly(out); + return; + } else { + MNodeService mnode = MNodeService.getInstance(request); + mnode.setSession(session); + QueryEngineDescription qed = mnode.getQueryEngineDescription(session, engine); + response.setContentType("text/xml"); + response.setStatus(200); + out = response.getOutputStream(); + TypeMarshaller.marshalTypeToOutputStream(qed, out); + IOUtils.closeQuietly(out); + return; + } + } + + + } catch (BaseException be) { + // report Exceptions as clearly as possible + try { + out = response.getOutputStream(); + } catch (IOException e) { + logMetacat.error("Could not get output stream from response", e); + } serializeException(be, out); } catch (Exception e) { // report Exceptions as clearly and generically as possible logMetacat.error(e.getClass() + ": " + e.getMessage(), e); - try { - out = response.getOutputStream(); - } catch (IOException ioe) { - logMetacat.error("Could not get output stream from response", ioe); - } - ServiceFailure se = new ServiceFailure("0000", e.getMessage()); + try { + out = response.getOutputStream(); + } catch (IOException ioe) { + logMetacat.error("Could not get output stream from response", ioe); + } + ServiceFailure se = new ServiceFailure("0000", e.getMessage()); serializeException(se, out); } } - + /* * Handle the solr query sent by the http post method */ private void doPostQuery(String engine) { OutputStream out = null; try { - // NOTE: we set the session explicitly for the MNode instance since these methods do not provide a parameter + // NOTE: we set the session explicitly for the MNode instance since these methods do not provide a parameter collectMultipartParams(); - MNodeService mnode = MNodeService.getInstance(request); - if (multipartparams == null || multipartparams.isEmpty()) { + MNodeService mnode = MNodeService.getInstance(request); + if(multipartparams == null || multipartparams.isEmpty()) { throw new InvalidRequest("2823", "The request doesn't have any query information by the HTTP POST method."); } HashMap params = new HashMap(); - for (String key : multipartparams.keySet()) { + for(String key : multipartparams.keySet()) { List values = multipartparams.get(key); - logMetacat.debug("MNResourceHandler.doPostQuery -the key " + key + " has the value " + values); - if (values != null) { + logMetacat.debug("MNResourceHandler.doPostQuery -the key "+key +" has the value "+values); + if(values != null) { String[] arrayValues = values.toArray(new String[0]); params.put(key, arrayValues); } @@ -675,115 +675,115 @@ private void doPostQuery(String engine) { serializeException(se, out); } } - + private void doViews(String format, String pid) { - - OutputStream out = null; - MNodeService mnode = MNodeService.getInstance(request); - - try { - // get a list of views - if (pid != null) { - long start = System.currentTimeMillis(); - Identifier identifier = new Identifier(); - identifier.setValue(pid); - InputStream stream = null; - try { - stream = mnode.view(session, format, identifier); - // set the content-type if we have it from the implementation - if (stream instanceof ContentTypeInputStream) { - response.setContentType(((ContentTypeInputStream) stream).getContentType()); - } - response.setStatus(200); - out = response.getOutputStream(); - // write the results to the output stream - IOUtils.copyLarge(stream, out); - } finally { - if (stream != null) { - IOUtils.closeQuietly(stream); - } + + OutputStream out = null; + MNodeService mnode = MNodeService.getInstance(request); + + try { + // get a list of views + if (pid != null) { + long start = System.currentTimeMillis(); + Identifier identifier = new Identifier(); + identifier.setValue(pid); + InputStream stream = null; + try { + stream = mnode.view(session, format, identifier); + // set the content-type if we have it from the implementation + if (stream instanceof ContentTypeInputStream) { + response.setContentType(((ContentTypeInputStream) stream).getContentType()); } - long end = System.currentTimeMillis(); - IOUtils.closeQuietly(out); - logMetacat.info(Settings.PERFORMANCELOG + pid + Settings.PERFORMANCELOG_VIEW_METHOD + " Total view method" + Settings.PERFORMANCELOG_DURATION + (end - start) / 1000); - return; - } else { - // TODO: list the registered views - //BaseException ni = new NotImplemented("9999", "MN.listViews() is not implemented at this node"); - //throw ni; - OptionList list = mnode.listViews(session); - - response.setContentType("text/xml"); response.setStatus(200); - TypeMarshaller.marshalTypeToOutputStream(list, response.getOutputStream()); - IOUtils.closeQuietly(response.getOutputStream()); - } - - - } catch (BaseException be) { - // report Exceptions as clearly as possible - try { out = response.getOutputStream(); - } catch (IOException e) { - logMetacat.error("Could not get output stream from response", e); - } + // write the results to the output stream + IOUtils.copyLarge(stream, out); + } finally { + if (stream != null) { + IOUtils.closeQuietly(stream); + } + } + long end = System.currentTimeMillis(); + IOUtils.closeQuietly(out); + logMetacat.info(Settings.PERFORMANCELOG + pid + Settings.PERFORMANCELOG_VIEW_METHOD + " Total view method" + Settings.PERFORMANCELOG_DURATION + (end-start)/1000); + return; + } else { + // TODO: list the registered views + //BaseException ni = new NotImplemented("9999", "MN.listViews() is not implemented at this node"); + //throw ni; + OptionList list = mnode.listViews(session); + + response.setContentType("text/xml"); + response.setStatus(200); + TypeMarshaller.marshalTypeToOutputStream(list, response.getOutputStream()); + IOUtils.closeQuietly(response.getOutputStream()); + } + + + } catch (BaseException be) { + // report Exceptions as clearly as possible + try { + out = response.getOutputStream(); + } catch (IOException e) { + logMetacat.error("Could not get output stream from response", e); + } serializeException(be, out); } catch (Exception e) { // report Exceptions as clearly and generically as possible logMetacat.error(e.getClass() + ": " + e.getMessage(), e); - try { - out = response.getOutputStream(); - } catch (IOException ioe) { - logMetacat.error("Could not get output stream from response", ioe); - } - ServiceFailure se = new ServiceFailure("0000", e.getMessage()); + try { + out = response.getOutputStream(); + } catch (IOException ioe) { + logMetacat.error("Could not get output stream from response", ioe); + } + ServiceFailure se = new ServiceFailure("0000", e.getMessage()); serializeException(se, out); } } - + /** * Handles notification of system metadata changes for the given identifier - * - * @param id the identifier for the object - * @throws InvalidToken - * @throws InvalidRequest - * @throws NotAuthorized - * @throws ServiceFailure - * @throws NotImplemented + * + * @param id the identifier for the object + * @throws InvalidToken + * @throws InvalidRequest + * @throws NotAuthorized + * @throws ServiceFailure + * @throws NotImplemented */ - private void systemMetadataChanged() - throws NotImplemented, ServiceFailure, NotAuthorized, InvalidRequest, - InvalidToken { - + private void systemMetadataChanged() + throws NotImplemented, ServiceFailure, NotAuthorized, InvalidRequest, + InvalidToken { + ReadOnlyChecker checker = new ReadOnlyChecker(); boolean isReadOnlyMode = checker.isReadOnly(); - if (isReadOnlyMode) { + if(isReadOnlyMode) { throw new ServiceFailure("1333", ReadOnlyChecker.DATAONEERROR); } - + //final long serialVersion = 0L; String serialVersionStr = null; String dateSysMetaLastModifiedStr = null; - + // mkae sure we have the multipart params try { - initMultipartParams(); - } catch (Exception e1) { - throw new ServiceFailure("1333", "Could not collect the multipart params for the request"); - } - + initMultipartParams(); + } catch (Exception e1) { + throw new ServiceFailure("1333", "Could not collect the multipart params for the request"); + } + // get the pid String id = null; try { - id = multipartparams.get("pid").get(0); + id = multipartparams.get("pid").get(0); } catch (NullPointerException e) { String msg = "The 'pid' must be provided as a parameter and was not."; logMetacat.error(msg); throw new InvalidRequest("1334", msg); - } + } final Identifier pid = new Identifier(); pid.setValue(id); - + // get the serialVersion try { serialVersionStr = multipartparams.get("serialVersion").get(0); @@ -791,26 +791,26 @@ private void systemMetadataChanged() String msg = "The 'serialVersion' must be provided as a parameter and was not."; logMetacat.error(msg); throw new InvalidRequest("1334", msg); - - } - + + } + final long serialVersion = (new Long(serialVersionStr)).longValue(); - + // get the dateSysMetaLastModified try { dateSysMetaLastModifiedStr = multipartparams.get("dateSysMetaLastModified").get(0); - - + + } catch (NullPointerException e) { - String msg = - "The 'dateSysMetaLastModified' must be provided as a " + - "parameter and was not, or was an invalid representation of the timestamp."; + String msg = + "The 'dateSysMetaLastModified' must be provided as a " + + "parameter and was not, or was an invalid representation of the timestamp."; logMetacat.error(msg); throw new InvalidRequest("1334", msg); - - } + + } final Date dateSysMetaLastModified = DateTimeMarshaller.deserializeDateToUTC(dateSysMetaLastModifiedStr); - + // check authorization before sending to implementation D1AuthHelper authDel = new D1AuthHelper(request, pid, "1331", "????"); authDel.doAdminAuthorization(session); @@ -820,7 +820,7 @@ private void systemMetadataChanged() // NotAuthorized failure = new NotAuthorized("1331", msg); // throw failure; // } - + // run it in a thread to avoid connection timeout final String ipAddress = request.getRemoteAddr(); final String userAgent = request.getHeader("User-Agent"); @@ -828,7 +828,7 @@ private void systemMetadataChanged() @Override public void run() { try { - // call the service + // call the service MNodeService.getInstance(request, ipAddress, userAgent).systemMetadataChanged(session, pid, serialVersion, dateSysMetaLastModified); } catch (Exception e) { logMetacat.error("Error running replication: " + e.getMessage(), e); @@ -838,44 +838,44 @@ public void run() { }; // submit the task, and that's it executor.submit(runner); - + // thread was started, so we return success response.setStatus(200); } - + /** * Handles identifier generation calls - * - * @throws InvalidRequest - * @throws NotImplemented - * @throws NotAuthorized - * @throws ServiceFailure - * @throws InvalidToken - * @throws IOException - * @throws MarshallingException + * + * @throws InvalidRequest + * @throws NotImplemented + * @throws NotAuthorized + * @throws ServiceFailure + * @throws InvalidToken + * @throws IOException + * @throws MarshallingException */ private void generateIdentifier() throws InvalidToken, ServiceFailure, NotAuthorized, NotImplemented, InvalidRequest, IOException, MarshallingException { - + // make sure we have the multipart params try { - initMultipartParams(); - } catch (Exception e1) { - throw new ServiceFailure("1333", "Could not collect the multipart params for the request"); - } - + initMultipartParams(); + } catch (Exception e1) { + throw new ServiceFailure("1333", "Could not collect the multipart params for the request"); + } + // get the scheme - String scheme = null; + String scheme = null; try { - scheme = multipartparams.get("scheme").get(0); + scheme = multipartparams.get("scheme").get(0); } catch (NullPointerException e) { String msg = "The 'scheme' parameter was not provided, using default"; logMetacat.warn(msg); } - + // get the fragment - String fragment = null; + String fragment = null; try { - fragment = multipartparams.get("fragment").get(0); + fragment = multipartparams.get("fragment").get(0); } catch (NullPointerException e) { String msg = "The 'fragment' parameter was not provided, using default"; logMetacat.warn(msg); @@ -892,7 +892,6 @@ private void generateIdentifier() throws InvalidToken, ServiceFailure, NotAuthor /** * Checks the access policy - * * @param id * @return * @throws ServiceFailure @@ -903,83 +902,83 @@ private void generateIdentifier() throws InvalidToken, ServiceFailure, NotAuthor * @throws InvalidRequest */ private boolean isAuthorized(String id) throws ServiceFailure, InvalidToken, NotFound, NotAuthorized, NotImplemented, InvalidRequest { - Identifier pid = new Identifier(); - pid.setValue(id); - Permission permission = null; - try { - String perm = params.get("action")[0]; - permission = Permission.convert(perm); - } catch (Exception e) { - logMetacat.warn("No permission specified"); - } - boolean result = MNodeService.getInstance(request).isAuthorized(session, pid, permission); - response.setStatus(200); - response.setContentType("text/xml"); - return result; + Identifier pid = new Identifier(); + pid.setValue(id); + Permission permission = null; + try { + String perm = params.get("action")[0]; + permission = Permission.convert(perm); + } catch (Exception e) { + logMetacat.warn("No permission specified"); + } + boolean result = MNodeService.getInstance(request).isAuthorized(session, pid, permission); + response.setStatus(200); + response.setContentType("text/xml"); + return result; } - + private void getToken() throws Exception { - - if (this.session != null) { - String userId = this.session.getSubject().getValue(); - String fullName = null; - try { - Person person = this.session.getSubjectInfo().getPerson(0); - fullName = person.getGivenName(0) + " " + person.getFamilyName(); - } catch (Exception e) { - logMetacat.warn(e.getMessage(), e); - } - - String token = null; - token = TokenGenerator.getInstance().getJWT(userId, fullName); - - response.setStatus(200); - response.setContentType("text/plain"); - OutputStream out = response.getOutputStream(); - out.write(token.getBytes(MetaCatServlet.DEFAULT_ENCODING)); - out.close(); - } else { - response.setStatus(401); - response.setContentType("text/plain"); - OutputStream out = response.getOutputStream(); - out.write("No session information found".getBytes(MetaCatServlet.DEFAULT_ENCODING)); - out.close(); - } - + + if (this.session != null) { + String userId = this.session.getSubject().getValue(); + String fullName = null; + try { + Person person = this.session.getSubjectInfo().getPerson(0); + fullName = person.getGivenName(0) + " " + person.getFamilyName(); + } catch (Exception e) { + logMetacat.warn(e.getMessage(), e); + } + + String token = null; + token = TokenGenerator.getInstance().getJWT(userId, fullName); + + response.setStatus(200); + response.setContentType("text/plain"); + OutputStream out = response.getOutputStream(); + out.write(token.getBytes(MetaCatServlet.DEFAULT_ENCODING)); + out.close(); + } else { + response.setStatus(401); + response.setContentType("text/plain"); + OutputStream out = response.getOutputStream(); + out.write("No session information found".getBytes(MetaCatServlet.DEFAULT_ENCODING)); + out.close(); + } + } - + private void whoami() throws Exception { - - if (this.session != null) { - Subject subject = this.session.getSubject(); - SubjectInfo subjectInfo = null; - try { - subjectInfo = this.session.getSubjectInfo(); - } catch (Exception e) { - logMetacat.warn(e.getMessage(), e); - } - - response.setStatus(200); - response.setContentType("text/plain"); - OutputStream out = response.getOutputStream(); - - if (subjectInfo != null) { - TypeMarshaller.marshalTypeToOutputStream(subjectInfo, out); - } else { - TypeMarshaller.marshalTypeToOutputStream(subject, out); - } - - out.close(); - } else { - response.setStatus(401); - response.setContentType("text/plain"); - OutputStream out = response.getOutputStream(); - out.write("No session information found".getBytes(MetaCatServlet.DEFAULT_ENCODING)); - out.close(); - } - + + if (this.session != null) { + Subject subject = this.session.getSubject(); + SubjectInfo subjectInfo = null; + try { + subjectInfo = this.session.getSubjectInfo(); + } catch (Exception e) { + logMetacat.warn(e.getMessage(), e); + } + + response.setStatus(200); + response.setContentType("text/plain"); + OutputStream out = response.getOutputStream(); + + if (subjectInfo != null) { + TypeMarshaller.marshalTypeToOutputStream(subjectInfo, out); + } else { + TypeMarshaller.marshalTypeToOutputStream(subject, out); + } + + out.close(); + } else { + response.setStatus(401); + response.setContentType("text/plain"); + OutputStream out = response.getOutputStream(); + out.write("No session information found".getBytes(MetaCatServlet.DEFAULT_ENCODING)); + out.close(); + } + } - + /** * Get the status of the system. Now we only support to get the size of the index queue. */ @@ -988,40 +987,38 @@ private void getStatus() throws IOException, NotAuthorized, ServiceFailure { response.setStatus(200); response.setContentType("text/xml"); OutputStream out = response.getOutputStream(); - out = response.getOutputStream(); + out = response.getOutputStream(); IOUtils.copy(result, out); IOUtils.closeQuietly(out); //IOUtils.copyLarge(result, out); } - + /** * Processes failed synchronization message - * * @throws NotImplemented * @throws ServiceFailure * @throws NotAuthorized * @throws InvalidRequest * @throws MarshallingException - * @throws IllegalAccessException - * @throws InstantiationException - * @throws IOException + * @throws IllegalAccessException + * @throws InstantiationException + * @throws IOException */ private void syncError() throws NotImplemented, ServiceFailure, NotAuthorized, InvalidRequest, MarshallingException, IOException, InstantiationException, IllegalAccessException { - SynchronizationFailed syncFailed = null; - try { - syncFailed = collectSynchronizationFailed(); - } catch (ParserConfigurationException e) { - throw new ServiceFailure("2161", e.getMessage()); - } catch (SAXException e) { - throw new ServiceFailure("2161", e.getMessage()); - } - - MNodeService.getInstance(request).synchronizationFailed(session, syncFailed); + SynchronizationFailed syncFailed = null; + try { + syncFailed = collectSynchronizationFailed(); + } catch (ParserConfigurationException e) { + throw new ServiceFailure("2161", e.getMessage()); + } catch (SAXException e) { + throw new ServiceFailure("2161", e.getMessage()); + } + + MNodeService.getInstance(request).synchronizationFailed(session, syncFailed); } - /** - * Calculate the checksum - * + /** + * Calculate the checksum * @throws NotImplemented * @throws MarshallingException * @throws IOException @@ -1032,23 +1029,23 @@ private void syncError() throws NotImplemented, ServiceFailure, NotAuthorized, I * @throws InvalidRequest */ private void checksum(String pid) throws NotImplemented, MarshallingException, IOException, InvalidToken, ServiceFailure, NotAuthorized, NotFound, InvalidRequest { - String checksumAlgorithm = "MD5"; - try { - checksumAlgorithm = PropertyService.getProperty("dataone.checksumAlgorithm.default"); - } catch (Exception e) { - logMetacat.warn("Could not lookup configured default checksum algorithm, using: " + checksumAlgorithm); - } - + String checksumAlgorithm = "MD5"; + try { + checksumAlgorithm = PropertyService.getProperty("dataone.checksumAlgorithm.default"); + } catch(Exception e) { + logMetacat.warn("Could not lookup configured default checksum algorithm, using: " + checksumAlgorithm); + } + Identifier pidid = new Identifier(); pidid.setValue(pid); try { checksumAlgorithm = params.get("checksumAlgorithm")[0]; - } catch (Exception e) { + } catch(Exception e) { //do nothing. use the default - logMetacat.warn("No algorithm specified, using default: " + checksumAlgorithm); + logMetacat.warn("No algorithm specified, using default: " + checksumAlgorithm); } logMetacat.debug("getting checksum for object " + pid + " with algorithm " + checksumAlgorithm); - + Checksum c = MNodeService.getInstance(request).getChecksum(session, pidid, checksumAlgorithm); logMetacat.debug("got checksum " + c.getValue()); response.setStatus(200); @@ -1056,43 +1053,42 @@ private void checksum(String pid) throws NotImplemented, MarshallingException, I TypeMarshaller.marshalTypeToOutputStream(c, response.getOutputStream()); logMetacat.debug("done serializing response."); IOUtils.closeQuietly(response.getOutputStream()); - + } - - /** + + /** * handle the replicate action for MN - * - * @throws MarshallingException - * @throws FileUploadException - * @throws IOException - * @throws InvalidRequest - * @throws ServiceFailure - * @throws UnsupportedType - * @throws InsufficientResources - * @throws NotAuthorized - * @throws NotImplemented - * @throws IllegalAccessException - * @throws InstantiationException - * @throws InvalidToken + * @throws MarshallingException + * @throws FileUploadException + * @throws IOException + * @throws InvalidRequest + * @throws ServiceFailure + * @throws UnsupportedType + * @throws InsufficientResources + * @throws NotAuthorized + * @throws NotImplemented + * @throws IllegalAccessException + * @throws InstantiationException + * @throws InvalidToken */ - private void replicate() - throws ServiceFailure, InvalidRequest, IOException, FileUploadException, - MarshallingException, NotImplemented, NotAuthorized, InsufficientResources, - UnsupportedType, InstantiationException, IllegalAccessException, InvalidToken { + private void replicate() + throws ServiceFailure, InvalidRequest, IOException, FileUploadException, + MarshallingException, NotImplemented, NotAuthorized, InsufficientResources, + UnsupportedType, InstantiationException, IllegalAccessException, InvalidToken { logMetacat.debug("in POST replicate()"); ReadOnlyChecker checker = new ReadOnlyChecker(); boolean isReadOnlyMode = checker.isReadOnly(); - if (isReadOnlyMode) { + if(isReadOnlyMode) { throw new ServiceFailure("2151", ReadOnlyChecker.DATAONEERROR); } - + // somewhat unorthodox, but the call is asynchronous and we'd like to return this info sooner boolean allowed = false; if (session == null) { - String msg = "No session was provided."; + String msg = "No session was provided."; NotAuthorized failure = new NotAuthorized("2152", msg); - throw failure; + throw failure; } else { // TODO: should we refactore replicate() in MNodeservice to not replicate, it would avoid a possible second listNodes call... D1AuthHelper authDel = new D1AuthHelper(request, null, "2152", "????"); @@ -1104,58 +1100,57 @@ private void replicate() // throw failure; // } } - + // parse the systemMetadata - Map files = collectMultipartFiles(); + Map files = collectMultipartFiles(); final SystemMetadata sysmeta = TypeMarshaller.unmarshalTypeFromFile(SystemMetadata.class, files.get("sysmeta")); - + String sn = multipartparams.get("sourceNode").get(0); logMetacat.debug("sourceNode: " + sn); final NodeReference sourceNode = new NodeReference(); sourceNode.setValue(sn); - + // run it in a thread to avoid connection timeout final String ipAddress = request.getRemoteAddr(); final String userAgent = request.getHeader("User-Agent"); Runnable runner = new Runnable() { - @Override - public void run() { - try { - MNodeService.getInstance(request, ipAddress, userAgent).replicate(session, sysmeta, sourceNode); - } catch (Exception e) { - logMetacat.error("Error running replication: " + e.getMessage(), e); - throw new RuntimeException(e.getMessage(), e); - } - } - }; - // submit the task, and that's it - executor.submit(runner); - - // thread was started, so we return success + @Override + public void run() { + try { + MNodeService.getInstance(request, ipAddress, userAgent).replicate(session, sysmeta, sourceNode); + } catch (Exception e) { + logMetacat.error("Error running replication: " + e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); + } + } + }; + // submit the task, and that's it + executor.submit(runner); + + // thread was started, so we return success response.setStatus(200); - + } /** * Handle the getReplica action for the MN - * - * @param id the identifier for the object - * @throws NotFound - * @throws ServiceFailure - * @throws NotImplemented - * @throws NotAuthorized - * @throws InvalidToken - * @throws InvalidRequest + * @param id the identifier for the object + * @throws NotFound + * @throws ServiceFailure + * @throws NotImplemented + * @throws NotAuthorized + * @throws InvalidToken + * @throws InvalidRequest */ - private void getReplica(String id) - throws InvalidRequest, InvalidToken, NotAuthorized, NotImplemented, - ServiceFailure, NotFound { - + private void getReplica(String id) + throws InvalidRequest, InvalidToken, NotAuthorized, NotImplemented, + ServiceFailure, NotFound { + Identifier pid = new Identifier(); pid.setValue(id); OutputStream out = null; InputStream dataBytes = null; - + try { // call the service dataBytes = MNodeService.getInstance(request).getReplica(session, pid); @@ -1170,47 +1165,47 @@ private void getReplica(String id) String msg = "There was an error writing the output: " + e.getMessage(); logMetacat.error(msg); throw new ServiceFailure("2181", msg); - + } } /** * Get the Node information - * + * * @throws MarshallingException * @throws IOException - * @throws InvalidRequest - * @throws ServiceFailure - * @throws NotAuthorized - * @throws NotImplemented + * @throws InvalidRequest + * @throws ServiceFailure + * @throws NotAuthorized + * @throws NotImplemented */ - private void node() - throws MarshallingException, IOException, NotImplemented, NotAuthorized, ServiceFailure, InvalidRequest { - + private void node() + throws MarshallingException, IOException, NotImplemented, NotAuthorized, ServiceFailure, InvalidRequest { + Node n = MNodeService.getInstance(request).getCapabilities(); - + response.setContentType("text/xml"); response.setStatus(200); TypeMarshaller.marshalTypeToOutputStream(n, response.getOutputStream()); IOUtils.closeQuietly(response.getOutputStream()); - + } - + /** * MN_crud.describe() * http://mule1.dataone.org/ArchitectureDocs/mn_api_crud.html#MN_crud.describe - * * @param pid - * @throws InvalidRequest - * @throws NotImplemented - * @throws NotFound - * @throws NotAuthorized - * @throws ServiceFailure - * @throws InvalidToken + * @throws InvalidRequest + * @throws NotImplemented + * @throws NotFound + * @throws NotAuthorized + * @throws ServiceFailure + * @throws InvalidToken */ - private void describeObject(String pid) throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented, InvalidRequest { - + private void describeObject(String pid) throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented, InvalidRequest + { + response.setContentType("text/xml"); Identifier id = new Identifier(); @@ -1218,18 +1213,18 @@ private void describeObject(String pid) throws InvalidToken, ServiceFailure, Not DescribeResponse dr = null; try { - dr = MNodeService.getInstance(request).describe(session, id); + dr = MNodeService.getInstance(request).describe(session, id); } catch (BaseException e) { - response.setStatus(e.getCode()); - response.addHeader("DataONE-Exception-Name", e.getClass().getName()); + response.setStatus(e.getCode()); + response.addHeader("DataONE-Exception-Name", e.getClass().getName()); response.addHeader("DataONE-Exception-DetailCode", e.getDetail_code()); response.addHeader("DataONE-Exception-Description", e.getDescription()); response.addHeader("DataONE-Exception-PID", id.getValue()); return; - } - + } + response.setStatus(200); - + //response.addHeader("pid", pid); response.addHeader("DataONE-Checksum", dr.getDataONE_Checksum().getAlgorithm() + "," + dr.getDataONE_Checksum().getValue()); response.addHeader("Content-Length", dr.getContent_Length() + ""); @@ -1237,180 +1232,184 @@ private void describeObject(String pid) throws InvalidToken, ServiceFailure, Not response.addHeader("DataONE-ObjectFormat", dr.getDataONE_ObjectFormatIdentifier().getValue()); response.addHeader("DataONE-SerialVersion", dr.getSerialVersion().toString()); - + } - + /** - * get the logs based on passed params. Available + * get the logs based on passed params. Available * See http://mule1.dataone.org/ArchitectureDocs/mn_api_crud.html#MN_crud.getLogRecords * for more info - * - * @throws NotImplemented - * @throws InvalidRequest - * @throws NotAuthorized - * @throws ServiceFailure - * @throws InvalidToken - * @throws IOException - * @throws MarshallingException + * @throws NotImplemented + * @throws InvalidRequest + * @throws NotAuthorized + * @throws ServiceFailure + * @throws InvalidToken + * @throws IOException + * @throws MarshallingException */ - private void getLog() throws InvalidToken, ServiceFailure, NotAuthorized, InvalidRequest, NotImplemented, IOException, MarshallingException { - + private void getLog() throws InvalidToken, ServiceFailure, NotAuthorized, InvalidRequest, NotImplemented, IOException, MarshallingException + { + Date fromDate = null; Date toDate = null; String event = null; Integer start = null; Integer count = null; String pidFilter = null; - + try { - String fromDateS = params.get("fromDate")[0]; + String fromDateS = params.get("fromDate")[0]; logMetacat.debug("param fromDateS: " + fromDateS); fromDate = DateTimeMarshaller.deserializeDateToUTC(fromDateS); } catch (Exception e) { - logMetacat.warn("Could not parse fromDate: " + e.getMessage()); + logMetacat.warn("Could not parse fromDate: " + e.getMessage()); } try { - String toDateS = params.get("toDate")[0]; + String toDateS = params.get("toDate")[0]; logMetacat.debug("param toDateS: " + toDateS); toDate = DateTimeMarshaller.deserializeDateToUTC(toDateS); } catch (Exception e) { - logMetacat.warn("Could not parse toDate: " + e.getMessage()); - } + logMetacat.warn("Could not parse toDate: " + e.getMessage()); + } try { - event = params.get("event")[0]; + event = params.get("event")[0]; } catch (Exception e) { - logMetacat.warn("Could not parse event: " + e.getMessage()); - } + logMetacat.warn("Could not parse event: " + e.getMessage()); + } logMetacat.debug("fromDate: " + fromDate + " toDate: " + toDate); - + try { - start = Integer.parseInt(params.get("start")[0]); + start = Integer.parseInt(params.get("start")[0]); } catch (Exception e) { - logMetacat.warn("Could not parse start: " + e.getMessage()); - } + logMetacat.warn("Could not parse start: " + e.getMessage()); + } try { - count = Integer.parseInt(params.get("count")[0]); + count = Integer.parseInt(params.get("count")[0]); } catch (Exception e) { - logMetacat.warn("Could not parse count: " + e.getMessage()); - } + logMetacat.warn("Could not parse count: " + e.getMessage()); + } + try { pidFilter = params.get("idFilter")[0]; } catch (Exception e) { logMetacat.warn("Could not parse pidFilter: " + e.getMessage()); } - + logMetacat.debug("calling getLogRecords"); Log log = MNodeService.getInstance(request).getLogRecords(session, fromDate, toDate, event, pidFilter, start, count); - + OutputStream out = response.getOutputStream(); response.setStatus(200); response.setContentType("text/xml"); - + TypeMarshaller.marshalTypeToOutputStream(log, out); IOUtils.closeQuietly(out); } - + + + /** * Implements REST version of DataONE CRUD API --> get - * * @param pid ID of data object to be read - * @throws NotImplemented - * @throws InvalidRequest - * @throws NotFound - * @throws NotAuthorized - * @throws ServiceFailure - * @throws InvalidToken - * @throws IOException - * @throws MarshallingException + * @throws NotImplemented + * @throws InvalidRequest + * @throws NotFound + * @throws NotAuthorized + * @throws ServiceFailure + * @throws InvalidToken + * @throws IOException + * @throws MarshallingException */ protected void getObject(String pid) throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, InvalidRequest, NotImplemented, IOException, MarshallingException { OutputStream out = null; - - if (pid != null) { //get a specific document + + if (pid != null) { //get a specific document long start = System.currentTimeMillis(); Identifier id = new Identifier(); id.setValue(pid); - + SystemMetadata sm = MNodeService.getInstance(request).getSystemMetadata(session, id); - + // set the headers for the content String mimeType = null; String charset = null; ObjectFormat objectFormat = null; - + try { - objectFormat = ObjectFormatCache.getInstance().getFormat(sm.getFormatId()); - } catch (BaseException be) { - logMetacat.warn("Could not lookup ObjectFormat for: " + sm.getFormatId(), be); - } + objectFormat = ObjectFormatCache.getInstance().getFormat(sm.getFormatId()); + } catch (BaseException be) { + logMetacat.warn("Could not lookup ObjectFormat for: " + sm.getFormatId(), be); + } // do we have mediaType/encoding in SM? MediaType mediaType = sm.getMediaType(); if (mediaType == null && objectFormat != null) { - try { - mediaType = objectFormat.getMediaType(); - } catch (Exception e) { - logMetacat.warn("Could not lookup MediaType for: " + sm.getFormatId(), e); - } + try { + mediaType = objectFormat.getMediaType(); + } catch (Exception e) { + logMetacat.warn("Could not lookup MediaType for: " + sm.getFormatId(), e); + } } if (mediaType != null) { mimeType = mediaType.getName(); if (mediaType.getPropertyList() != null) { - Iterator iter = mediaType.getPropertyList().iterator(); - while (iter.hasNext()) { - MediaTypeProperty mtp = iter.next(); - if (mtp.getName().equalsIgnoreCase("charset")) { - charset = mtp.getValue(); - mimeType += "; charset=" + charset; - break; - } - } + Iterator iter = mediaType.getPropertyList().iterator(); + while (iter.hasNext()) { + MediaTypeProperty mtp = iter.next(); + if (mtp.getName().equalsIgnoreCase("charset")) { + charset = mtp.getValue(); + mimeType += "; charset=" + charset; + break; + } + } } } // check object format - + // use the fallback from v1 impl if (mimeType == null) { - mimeType = ObjectFormatInfo.instance().getMimeType(sm.getFormatId().getValue()); - - // still null? - if (mimeType == null) { - mimeType = "application/octet-stream"; - } + mimeType = ObjectFormatInfo.instance().getMimeType(sm.getFormatId().getValue()); + + // still null? + if (mimeType == null) { + mimeType = "application/octet-stream"; + } } - + // check for filename in SM first String filename = sm.getFileName(); // then fallback to using id and extension if (filename == null) { - String extension = null; - if (objectFormat != null) { - extension = objectFormat.getExtension(); - } - if (extension == null) { - extension = ObjectFormatInfo.instance().getExtension(sm.getFormatId().getValue()); - } - filename = id.getValue(); - if (extension != null && filename != null && !filename.endsWith(extension)) { - filename = id.getValue() + "." + extension; - } + String extension = null; + if(objectFormat != null) { + extension = objectFormat.getExtension(); + } + if (extension == null) { + extension = ObjectFormatInfo.instance().getExtension(sm.getFormatId().getValue()); + } + filename = id.getValue(); + if (extension != null && filename != null && !filename.endsWith(extension)) { + filename = id.getValue() + "." + extension; + } } response.setContentType(mimeType); - response.setHeader("Content-Disposition", "inline; filename=\"" + filename + "\""); + response.setHeader("Content-Disposition", "inline; filename=\"" + filename+"\""); InputStream data = null; try { data = MNodeService.getInstance(request).get(session, id); - out = response.getOutputStream(); + out = response.getOutputStream(); response.setStatus(200); IOUtils.copyLarge(data, out); IOUtils.closeQuietly(out); } finally { if (data != null) { - IOUtils.closeQuietly(data); + IOUtils.closeQuietly(data); } } long end = System.currentTimeMillis(); - logMetacat.info(Settings.PERFORMANCELOG + pid + Settings.PERFORMANCELOG_GET_METHOD + " Total get method" + Settings.PERFORMANCELOG_DURATION + (end - start) / 1000); - } else { //call listObjects with specified params + logMetacat.info(Settings.PERFORMANCELOG + pid + Settings.PERFORMANCELOG_GET_METHOD + " Total get method" + Settings.PERFORMANCELOG_DURATION + (end-start)/1000); + } + else + { //call listObjects with specified params Date startTime = null; Date endTime = null; ObjectFormatIdentifier formatId = null; @@ -1420,127 +1419,149 @@ protected void getObject(String pid) throws InvalidToken, ServiceFailure, NotAut //TODO: make the max count into a const int count = 1000; Enumeration paramlist = request.getParameterNames(); - while (paramlist.hasMoreElements()) { //parse the params and make the crud call + while (paramlist.hasMoreElements()) + { //parse the params and make the crud call String name = (String) paramlist.nextElement(); - String[] value = (String[]) request.getParameterValues(name); - - if (name.equals("fromDate") && value != null) { - try { - //startTime = dateFormat.parse(value[0]); - startTime = DateTimeMarshaller.deserializeDateToUTC(value[0]); + String[] value = (String[])request.getParameterValues(name); + + if (name.equals("fromDate") && value != null) + { + try + { + //startTime = dateFormat.parse(value[0]); + startTime = DateTimeMarshaller.deserializeDateToUTC(value[0]); //startTime = parseDateAndConvertToGMT(value[0]); - } catch (Exception e) { //if we can't parse it, just don't use the fromDate param + } + catch(Exception e) + { //if we can't parse it, just don't use the fromDate param logMetacat.warn("Could not parse fromDate: " + value[0], e); - throw new InvalidRequest("1540", "Could not parse fromDate: " + value[0] + " since " + e.getMessage()); + throw new InvalidRequest("1540", "Could not parse fromDate: " + value[0]+" since "+e.getMessage()); //startTime = null; } - } else if (name.equals("toDate") && value != null) { - try { - endTime = DateTimeMarshaller.deserializeDateToUTC(value[0]); - } catch (Exception e) { //if we can't parse it, just don't use the toDate param + } + else if(name.equals("toDate") && value != null) + { + try + { + endTime = DateTimeMarshaller.deserializeDateToUTC(value[0]); + } + catch(Exception e) + { //if we can't parse it, just don't use the toDate param logMetacat.warn("Could not parse toDate: " + value[0], e); - throw new InvalidRequest("1540", "Could not parse toDate: " + value[0] + " since " + e.getMessage()); + throw new InvalidRequest("1540", "Could not parse toDate: " + value[0]+" since "+e.getMessage()); //endTime = null; } - } else if (name.equals("formatId") && value != null) { - formatId = new ObjectFormatIdentifier(); - formatId.setValue(value[0]); - } else if (name.equals("identifier") && value != null) { - identifier = new Identifier(); - identifier.setValue(value[0]); - } else if (name.equals("replicaStatus") && value != null) { - if (value != null && - value.length > 0 && - (value[0].equalsIgnoreCase("false") || value[0].equalsIgnoreCase("no"))) { + } + else if(name.equals("formatId") && value != null) + { + formatId = new ObjectFormatIdentifier(); + formatId.setValue(value[0]); + } + else if(name.equals("identifier") && value != null) + { + identifier = new Identifier(); + identifier.setValue(value[0]); + } + else if(name.equals("replicaStatus") && value != null) + { + if(value != null && + value.length > 0 && + (value[0].equalsIgnoreCase("false") || value[0].equalsIgnoreCase("no"))) + { replicaStatus = false; } - } else if (name.equals("start") && value != null) { + } + else if(name.equals("start") && value != null) + { start = new Integer(value[0]).intValue(); - } else if (name.equals("count") && value != null) { + } + else if(name.equals("count") && value != null) + { count = new Integer(value[0]).intValue(); } } //make the crud call logMetacat.debug("session: " + session + " startTime: " + startTime + - " endTime: " + endTime + " formatId: " + - formatId + " replicaStatus: " + replicaStatus + + " endTime: " + endTime + " formatId: " + + formatId + " replicaStatus: " + replicaStatus + " start: " + start + " count: " + count); - - ObjectList ol = - MNodeService.getInstance(request).listObjects(session, startTime, endTime, - formatId, identifier, replicaStatus, start, count); - - out = response.getOutputStream(); + + ObjectList ol = + MNodeService.getInstance(request).listObjects(session, startTime, endTime, + formatId, identifier, replicaStatus, start, count); + + out = response.getOutputStream(); response.setStatus(200); response.setContentType("text/xml"); // Serialize and write it to the output stream TypeMarshaller.marshalTypeToOutputStream(ol, out); IOUtils.closeQuietly(out); } - + } - + /** * Retrieve data package as Bagit zip - * - * @param format - * @param pid The pid of the resource map defining the pacakage - * @throws NotImplemented - * @throws NotFound - * @throws NotAuthorized - * @throws ServiceFailure - * @throws InvalidToken - * @throws IOException - * @throws InvalidRequest + * @param pid + * @throws NotImplemented + * @throws NotFound + * @throws NotAuthorized + * @throws ServiceFailure + * @throws InvalidToken + * @throws IOException + * @throws InvalidRequest */ protected void getPackage(String format, String pid) throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented, IOException, InvalidRequest { - long start = System.currentTimeMillis(); Identifier id = new Identifier(); id.setValue(pid); ObjectFormatIdentifier formatId = null; if (format != null) { - formatId = new ObjectFormatIdentifier(); - formatId.setValue(format); + formatId = new ObjectFormatIdentifier(); + formatId.setValue(format); } InputStream is = null; try { is = MNodeService.getInstance(request).getPackage(session, formatId , id); - //Use the pid as the file name prefix, replacing all non-word characters - String filename = pid.replaceAll("\\W", "_") + ".zip"; - response.setHeader("Content-Disposition", "inline; filename=\"" + filename + "\""); + + //Use the pid as the file name prefix, replacing all non-word characters + String filename = pid.replaceAll("\\W", "_"); + + response.setHeader("Content-Disposition", "inline; filename=\"" + filename+"\""); response.setContentType("application/zip"); response.setStatus(200); OutputStream out = response.getOutputStream(); - + // write it to the output stream IOUtils.copyLarge(is, out); + IOUtils.closeQuietly(out); long end = System.currentTimeMillis(); - logMetacat.info(Settings.PERFORMANCELOG + pid + Settings.PERFORMANCELOG_GET_PACKAGE_METHOD + " Total getPackage method" + Settings.PERFORMANCELOG_DURATION + (end - start) / 1000); + logMetacat.info(Settings.PERFORMANCELOG + pid + Settings.PERFORMANCELOG_GET_PACKAGE_METHOD + " Total getPackage method" + Settings.PERFORMANCELOG_DURATION + (end-start)/1000); + } finally { IOUtils.closeQuietly(is); } - } - - protected void publish(String pid) throws InvalidToken, ServiceFailure, - NotAuthorized, NotFound, NotImplemented, IOException, - MarshallingException, InvalidRequest, IdentifierNotUnique, - UnsupportedType, InsufficientResources, InvalidSystemMetadata { - - // publish the object - Identifier originalIdentifier = new Identifier(); - originalIdentifier.setValue(pid); - Identifier newIdentifier = MNodeService.getInstance(request).publish(session, originalIdentifier); - - response.setStatus(200); - response.setContentType("text/xml"); - OutputStream out = response.getOutputStream(); - - // write new identifier to the output stream - TypeMarshaller.marshalTypeToOutputStream(newIdentifier, out); - IOUtils.closeQuietly(out); - } + } + + protected void publish(String pid) throws InvalidToken, ServiceFailure, + NotAuthorized, NotFound, NotImplemented, IOException, + MarshallingException, InvalidRequest, IdentifierNotUnique, + UnsupportedType, InsufficientResources, InvalidSystemMetadata { + + // publish the object + Identifier originalIdentifier = new Identifier(); + originalIdentifier.setValue(pid); + Identifier newIdentifier = MNodeService.getInstance(request).publish(session, originalIdentifier); + + response.setStatus(200); + response.setContentType("text/xml"); + OutputStream out = response.getOutputStream(); + + // write new identifier to the output stream + TypeMarshaller.marshalTypeToOutputStream(newIdentifier, out); + IOUtils.closeQuietly(out); + } /** * Retrieve System Metadata @@ -1732,63 +1753,63 @@ private void archive(String pid) throws InvalidToken, ServiceFailure, NotAuthori logMetacat.info(Settings.PERFORMANCELOG + pid + Settings.PERFORMANCELOG_ARCHIVE_METHOD + " Total archive method" + Settings.PERFORMANCELOG_DURATION + (end-start)/1000); } - protected SynchronizationFailed collectSynchronizationFailed() throws IOException, ServiceFailure, InvalidRequest, MarshallingException, InstantiationException, IllegalAccessException, ParserConfigurationException, SAXException { - - // Read the incoming data from its Mime Multipart encoding - logMetacat.debug("Disassembling MIME multipart form"); - InputStream sf = null; - - // handle MMP inputs - File tmpDir = getTempDirectory(); - logMetacat.debug("temp dir: " + tmpDir.getAbsolutePath()); - MultipartRequestResolver mrr = - new MultipartRequestResolver(tmpDir.getAbsolutePath(), MAX_UPLOAD_SIZE, 0); - MultipartRequest mr = null; - try { - mr = mrr.resolveMultipart(request); - } catch (Exception e) { - throw new ServiceFailure("2161", - "Could not resolve multipart: " + e.getMessage()); - } - logMetacat.debug("resolved multipart request"); - Map files = mr.getMultipartFiles(); - if (files == null || files.keySet() == null) { - throw new InvalidRequest("2163", - "must have multipart file with name 'message'"); - } - logMetacat.debug("got multipart files"); - - multipartparams = mr.getMultipartParameters(); - - File sfFile = files.get("message"); - if (sfFile == null) { - throw new InvalidRequest("2163", - "Missing the required file-part 'message' from the multipart request."); - } - logMetacat.debug("sfFile: " + sfFile.getAbsolutePath()); - sf = new FileInputStream(sfFile); - - SynchronizationFailed syncFailed = (SynchronizationFailed) ExceptionHandler.deserializeXml(sf, "Error deserializing exception"); - return syncFailed; - } - - /** - * Update the system metadata for a specified identifier - * @throws ServiceFailure - * @throws InvalidRequest - * @throws InstantiationException - * @throws IllegalAccessException - * @throws IOException - * @throws MarshallingException - * @throws NotImplemented - * @throws NotAuthorized - * @throws InvalidSystemMetadata - * @throws InvalidToken - */ - protected void updateSystemMetadata() throws ServiceFailure, InvalidRequest, - InstantiationException, IllegalAccessException, IOException, MarshallingException, NotImplemented, - NotAuthorized, InvalidSystemMetadata, InvalidToken { - // Read the incoming data from its Mime Multipart encoding + protected SynchronizationFailed collectSynchronizationFailed() throws IOException, ServiceFailure, InvalidRequest, MarshallingException, InstantiationException, IllegalAccessException, ParserConfigurationException, SAXException { + + // Read the incoming data from its Mime Multipart encoding + logMetacat.debug("Disassembling MIME multipart form"); + InputStream sf = null; + + // handle MMP inputs + File tmpDir = getTempDirectory(); + logMetacat.debug("temp dir: " + tmpDir.getAbsolutePath()); + MultipartRequestResolver mrr = + new MultipartRequestResolver(tmpDir.getAbsolutePath(), MAX_UPLOAD_SIZE, 0); + MultipartRequest mr = null; + try { + mr = mrr.resolveMultipart(request); + } catch (Exception e) { + throw new ServiceFailure("2161", + "Could not resolve multipart: " + e.getMessage()); + } + logMetacat.debug("resolved multipart request"); + Map files = mr.getMultipartFiles(); + if (files == null || files.keySet() == null) { + throw new InvalidRequest("2163", + "must have multipart file with name 'message'"); + } + logMetacat.debug("got multipart files"); + + multipartparams = mr.getMultipartParameters(); + + File sfFile = files.get("message"); + if (sfFile == null) { + throw new InvalidRequest("2163", + "Missing the required file-part 'message' from the multipart request."); + } + logMetacat.debug("sfFile: " + sfFile.getAbsolutePath()); + sf = new FileInputStream(sfFile); + + SynchronizationFailed syncFailed = (SynchronizationFailed) ExceptionHandler.deserializeXml(sf, "Error deserializing exception"); + return syncFailed; + } + + /** + * Update the system metadata for a specified identifier + * @throws ServiceFailure + * @throws InvalidRequest + * @throws InstantiationException + * @throws IllegalAccessException + * @throws IOException + * @throws MarshallingException + * @throws NotImplemented + * @throws NotAuthorized + * @throws InvalidSystemMetadata + * @throws InvalidToken + */ + protected void updateSystemMetadata() throws ServiceFailure, InvalidRequest, + InstantiationException, IllegalAccessException, IOException, MarshallingException, NotImplemented, + NotAuthorized, InvalidSystemMetadata, InvalidToken { + // Read the incoming data from its Mime Multipart encoding Map files = collectMultipartFiles(); // get the encoded pid string from the body and make the object @@ -1806,6 +1827,6 @@ protected void updateSystemMetadata() throws ServiceFailure, InvalidRequest, logMetacat.debug("updating system metadata with pid " + pid.getValue()); MNodeService.getInstance(request).updateSystemMetadata(session, pid, systemMetadata); - } + } } From d6a751f78c977f1370b1cb26f861f9e8eb2eda93 Mon Sep 17 00:00:00 2001 From: Jing Tao Date: Thu, 22 Jul 2021 09:26:48 -0700 Subject: [PATCH 20/45] Remove an import statement which imported a deleted class. --- src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java b/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java index cf3431132..3db914898 100644 --- a/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java +++ b/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java @@ -100,7 +100,6 @@ import edu.ucsb.nceas.metacat.restservice.multipart.DetailedFileInputStream; import edu.ucsb.nceas.metacat.restservice.multipart.MultipartRequestWithSysmeta; import edu.ucsb.nceas.metacat.restservice.multipart.StreamingMultipartRequestResolver; -import edu.ucsb.nceas.metacat.util.DeleteOnCloseFileInputStream; import edu.ucsb.nceas.utilities.PropertyNotFoundException; /** From 352ed9afeab08b7f7a01ae323b6c8f1bd7242477 Mon Sep 17 00:00:00 2001 From: Thomas Thelen Date: Thu, 22 Jul 2021 10:12:05 -0700 Subject: [PATCH 21/45] Fix typo in import --- .../ucsb/nceas/metacat/dataone/D1NodeVersionCheckerTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/edu/ucsb/nceas/metacat/dataone/D1NodeVersionCheckerTest.java b/test/edu/ucsb/nceas/metacat/dataone/D1NodeVersionCheckerTest.java index bc52b3819..e49c70eb6 100644 --- a/test/edu/ucsb/nceas/metacat/dataone/D1NodeVersionCheckerTest.java +++ b/test/edu/ucsb/nceas/metacat/dataone/D1NodeVersionCheckerTest.java @@ -33,7 +33,7 @@ import edu.ucsb.nceas.metacat.dataone.MNodeService; import edu.ucsb.nceas.metacat.properties.SkinPropertyService; import edu.ucsb.nceas.metacat.service.ServiceService; -import edu.ucsb.nceas.utilities.IOUtil;s +import edu.ucsb.nceas.utilities.IOUtil; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; From dd600a9f94ae5f999227a4fc2389c61c3ff1e43e Mon Sep 17 00:00:00 2001 From: thomas Date: Thu, 22 Jul 2021 14:40:02 -0700 Subject: [PATCH 22/45] Refactor package download test --- .../metacat/dataone/MNodeServiceTest.java | 202 ++++++++---------- 1 file changed, 93 insertions(+), 109 deletions(-) diff --git a/test/edu/ucsb/nceas/metacat/dataone/MNodeServiceTest.java b/test/edu/ucsb/nceas/metacat/dataone/MNodeServiceTest.java index 0e5c3859a..bdf6edf42 100644 --- a/test/edu/ucsb/nceas/metacat/dataone/MNodeServiceTest.java +++ b/test/edu/ucsb/nceas/metacat/dataone/MNodeServiceTest.java @@ -59,11 +59,15 @@ import java.util.ArrayList; import java.util.Calendar; import java.util.Date; +import java.util.Enumeration; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Vector; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; +import java.util.zip.ZipInputStream; import junit.framework.Test; import junit.framework.TestSuite; @@ -1798,114 +1802,100 @@ public void testGetPackage() { */ public void testGetOREPackage() { printTestHeader("testGetOREPackage"); - try { - - // construct the ORE package - Identifier resourceMapId = new Identifier(); - //resourceMapId.setValue("doi://1234/AA/map.1.1"); - resourceMapId.setValue("testGetOREPackage." + System.currentTimeMillis()); - Identifier metadataId = new Identifier(); - metadataId.setValue("doi://1234/AA/meta.1." + + System.currentTimeMillis()); - List dataIds = new ArrayList(); - Identifier dataId = new Identifier(); - dataId.setValue("doi://1234/AA/data.1." + System.currentTimeMillis()); - Identifier dataId2 = new Identifier(); - dataId2.setValue("doi://1234/AA/data.2." + System.currentTimeMillis()); - dataIds.add(dataId); - dataIds.add(dataId2); - Map> idMap = new HashMap>(); - idMap.put(metadataId, dataIds); - ResourceMapFactory rmf = ResourceMapFactory.getInstance(); - ResourceMap resourceMap = rmf.createResourceMap(resourceMapId, idMap); - assertNotNull(resourceMap); - String rdfXml = ResourceMapFactory.getInstance().serializeResourceMap(resourceMap); - assertNotNull(rdfXml); - - Session session = getTestSession(); - InputStream object = null; - SystemMetadata sysmeta = null; - - // save the data objects (data just contains their ID) - InputStream dataObject1 = new ByteArrayInputStream(dataId.getValue().getBytes("UTF-8")); - sysmeta = createSystemMetadata(dataId, session.getSubject(), dataObject1); - MNodeService.getInstance(request).create(session, dataId, dataObject1, sysmeta); - // second data file - InputStream dataObject2 = new ByteArrayInputStream(dataId2.getValue().getBytes("UTF-8")); - sysmeta = createSystemMetadata(dataId2, session.getSubject(), dataObject2); - MNodeService.getInstance(request).create(session, dataId2, dataObject2, sysmeta); - // metadata file - InputStream metadataObject = new ByteArrayInputStream(metadataId.getValue().getBytes("UTF-8")); - sysmeta = createSystemMetadata(metadataId, session.getSubject(), metadataObject); - MNodeService.getInstance(request).create(session, metadataId, metadataObject, sysmeta); - - // save the ORE object - Thread.sleep(10000); - object = new ByteArrayInputStream(rdfXml.getBytes("UTF-8")); - sysmeta = createSystemMetadata(resourceMapId, session.getSubject(), object); - sysmeta.setFormatId(ObjectFormatCache.getInstance().getFormat("http://www.openarchives.org/ore/terms").getFormatId()); - Identifier pid = MNodeService.getInstance(request).create(session, resourceMapId, object, sysmeta); - - // get the package we uploaded - ObjectFormatIdentifier format = new ObjectFormatIdentifier(); - format.setValue("application/bagit-097"); - InputStream bagStream = MNodeService.getInstance(request).getPackage(session, format, pid); - File bagFile = File.createTempFile("bagit.", ".zip"); - IOUtils.copy(bagStream, new FileOutputStream(bagFile)); - BagFactory bagFactory = new BagFactory(); - Bag bag = bagFactory.createBag(bagFile); - Iterator manifestIter = bag.getTagManifests().iterator(); - while (manifestIter.hasNext()) { - String filepath = manifestIter.next().getFilepath(); - BagFile entryFile = bag.getBagFile(filepath); - InputStream result = entryFile.newInputStream(); - // check ORE - if (filepath.contains(resourceMapId.getValue())) { - object.reset(); - assertTrue(object.available() > 0); - assertTrue(result.available() > 0); - assertTrue(IOUtils.contentEquals(result, object)); - } - // check metadata - if (filepath.contains(metadataId.getValue())) { - metadataObject.reset(); - assertTrue(metadataObject.available() > 0); - assertTrue(result.available() > 0); - assertTrue(IOUtils.contentEquals(result, metadataObject)); - } - if (filepath.contains(dataId.getValue())) { - dataObject1.reset(); - assertTrue(dataObject1.available() > 0); - assertTrue(result.available() > 0); - assertTrue(IOUtils.contentEquals(result, dataObject1)); - } - if (filepath.contains(dataId2.getValue())) { - dataObject2.reset(); - assertTrue(dataObject2.available() > 0); - assertTrue(result.available() > 0); - assertTrue(IOUtils.contentEquals(result, dataObject2)); - } - - - } - - // clean up - bagFile.delete(); - - // test the ORE lookup - Thread.sleep(30000); - System.out.println("+++++++++++++++++++ the metadataId on the ore package is "+metadataId.getValue()); - List oreIds = MNodeService.getInstance(request).lookupOreFor(null, metadataId, true); - assertTrue(oreIds.contains(resourceMapId)); + // construct the ORE package + Identifier resourceMapId = new Identifier(); + //resourceMapId.setValue("doi://1234/AA/map.1.1"); + resourceMapId.setValue("testGetOREPackage." + System.currentTimeMillis()); + Identifier metadataId = new Identifier(); + metadataId.setValue("doi://1234/AA/meta.1." + +System.currentTimeMillis()); + List dataIds = new ArrayList(); + Identifier dataId = new Identifier(); + dataId.setValue("doi://1234/AA/data.1." + System.currentTimeMillis()); + Identifier dataId2 = new Identifier(); + dataId2.setValue("doi://1234/AA/data.2." + System.currentTimeMillis()); + dataIds.add(dataId); + dataIds.add(dataId2); + Map> idMap = new HashMap>(); + idMap.put(metadataId, dataIds); + ResourceMapFactory rmf = ResourceMapFactory.getInstance(); + ResourceMap resourceMap = rmf.createResourceMap(resourceMapId, idMap); + assertNotNull(resourceMap); + String rdfXml = ResourceMapFactory.getInstance().serializeResourceMap(resourceMap); + assertNotNull(rdfXml); - } catch (Exception e) { + Session session = getTestSession(); + InputStream object = null; + SystemMetadata sysmeta = null; + + // save the data objects (data just contains their ID) + InputStream dataObject1 = new ByteArrayInputStream(dataId.getValue().getBytes("UTF-8")); + sysmeta = createSystemMetadata(dataId, session.getSubject(), dataObject1); + MNodeService.getInstance(request).create(session, dataId, dataObject1, sysmeta); + // second data file + InputStream dataObject2 = new ByteArrayInputStream(dataId2.getValue().getBytes("UTF-8")); + sysmeta = createSystemMetadata(dataId2, session.getSubject(), dataObject2); + MNodeService.getInstance(request).create(session, dataId2, dataObject2, sysmeta); + // metadata file + InputStream metadataObject = new ByteArrayInputStream(metadataId.getValue().getBytes("UTF-8")); + sysmeta = createSystemMetadata(metadataId, session.getSubject(), metadataObject); + MNodeService.getInstance(request).create(session, metadataId, metadataObject, sysmeta); + + // save the ORE object + Thread.sleep(10000); + object = new ByteArrayInputStream(rdfXml.getBytes("UTF-8")); + sysmeta = createSystemMetadata(resourceMapId, session.getSubject(), object); + sysmeta.setFormatId(ObjectFormatCache.getInstance().getFormat("http://www.openarchives.org/ore/terms").getFormatId()); + Identifier pid = MNodeService.getInstance(request).create(session, resourceMapId, object, sysmeta); + + // get the package we uploaded + ObjectFormatIdentifier format = new ObjectFormatIdentifier(); + format.setValue("application/bagit-097"); + InputStream bagStream = MNodeService.getInstance(request).getPackage(session, format, pid); + File bagFile = File.createTempFile("bagit.", ".zip"); + IOUtils.copy(bagStream, new FileOutputStream(bagFile)); + // Check that the resource map is the same + String bagPath = bagFile.getAbsolutePath(); + ZipFile zipFile = new ZipFile(bagPath); + + Enumeration entries = zipFile.entries(); + + while (entries.hasMoreElements()) { + ZipEntry entry = entries.nextElement(); + // Check if it's the ORE + if (entry.getName().contains("testGetOREPackage")) { + InputStream stream = zipFile.getInputStream(entry); + object.reset(); + assertTrue(IOUtils.contentEquals(stream, object)); + } + // Check if it's the science metadata + else if (entry.getName().contains("meta.1")) { + InputStream stream = zipFile.getInputStream(entry); + metadataObject.reset(); + assertTrue(IOUtils.contentEquals(stream, metadataObject)); + } + // Check if it's the first data file + else if (entry.getName().contains("data.1")) { + InputStream stream = zipFile.getInputStream(entry); + dataObject1.reset(); + assertTrue(IOUtils.contentEquals(stream, dataObject1)); + } + // Check if it's the second data file + else if (entry.getName().contains("data.2")) { + InputStream stream = zipFile.getInputStream(entry); + dataObject2.reset(); + assertTrue(IOUtils.contentEquals(stream, dataObject2)); + } + } + // clean up + bagFile.delete(); + } + catch (Exception e) { e.printStackTrace(); fail("Unexpected error: " + e.getMessage()); } } - - /** * Test to publish a package */ @@ -1962,15 +1952,8 @@ public void testPublishPackage() { Thread.sleep(30000); List oreId3 = MNodeService.getInstance(request).lookupOreFor(session, dataId, true); assertTrue(oreId3.size() == 1); - //publish the package + //publish the package Identifier doi = MNodeService.getInstance(request).publish(session, metadataId); - // test the ORE lookup - Thread.sleep(30000); - System.out.println("+++++++++++++++++++ the metadataId on the ore package is "+metadataId.getValue()); - List oreIds = MNodeService.getInstance(request).lookupOreFor(session, doi, true); - assertTrue(oreIds.size() == 1); - List oreId2 = MNodeService.getInstance(request).lookupOreFor(session, dataId, true); - assertTrue(oreId2.size() == 2); } catch (Exception e) { e.printStackTrace(); @@ -3846,3 +3829,4 @@ private ResultSet getEventLogs(Identifier guid) throws Exception { return result; } } + From be174e1561939a9d55fdcd95807fe863a763df00 Mon Sep 17 00:00:00 2001 From: thomas Date: Thu, 22 Jul 2021 15:37:49 -0700 Subject: [PATCH 23/45] Add zip extension to downloaded packages --- .../ucsb/nceas/metacat/restservice/v1/MNResourceHandler.java | 3 ++- .../ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/edu/ucsb/nceas/metacat/restservice/v1/MNResourceHandler.java b/src/edu/ucsb/nceas/metacat/restservice/v1/MNResourceHandler.java index 74409483a..079e099a0 100644 --- a/src/edu/ucsb/nceas/metacat/restservice/v1/MNResourceHandler.java +++ b/src/edu/ucsb/nceas/metacat/restservice/v1/MNResourceHandler.java @@ -1239,7 +1239,7 @@ protected void getPackage(String pid) throws InvalidToken, ServiceFailure, NotAu InputStream is = MNodeService.getInstance(request).getPackage(session, null, id); //Use the pid as the file name prefix, replacing all non-word characters - String filename = pid.replaceAll("\\W", "_"); + String filename = pid.replaceAll("\\W", "_") + ".zip"; response.setHeader("Content-Disposition", "inline; filename=\"" + filename+"\""); response.setContentType("application/zip"); @@ -1492,3 +1492,4 @@ protected SynchronizationFailed collectSynchronizationFailed() throws IOExceptio } } + diff --git a/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java b/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java index 3db914898..a2765866b 100644 --- a/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java +++ b/src/edu/ucsb/nceas/metacat/restservice/v2/MNResourceHandler.java @@ -1525,7 +1525,7 @@ protected void getPackage(String format, String pid) throws InvalidToken, Servic is = MNodeService.getInstance(request).getPackage(session, formatId , id); //Use the pid as the file name prefix, replacing all non-word characters - String filename = pid.replaceAll("\\W", "_"); + String filename = pid.replaceAll("\\W", "_") + ".zip"; response.setHeader("Content-Disposition", "inline; filename=\"" + filename+"\""); response.setContentType("application/zip"); @@ -1829,3 +1829,4 @@ protected void updateSystemMetadata() throws ServiceFailure, InvalidRequest, } } + From d474ee6ce246581b324c8a832f59eb26ac9d146b Mon Sep 17 00:00:00 2001 From: thomas Date: Thu, 22 Jul 2021 15:53:44 -0700 Subject: [PATCH 24/45] Add removed tests --- .../edu/ucsb/nceas/metacat/dataone/MNodeServiceTest.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/edu/ucsb/nceas/metacat/dataone/MNodeServiceTest.java b/test/edu/ucsb/nceas/metacat/dataone/MNodeServiceTest.java index bdf6edf42..977f75a7d 100644 --- a/test/edu/ucsb/nceas/metacat/dataone/MNodeServiceTest.java +++ b/test/edu/ucsb/nceas/metacat/dataone/MNodeServiceTest.java @@ -1889,13 +1889,20 @@ else if (entry.getName().contains("data.2")) { } // clean up bagFile.delete(); + Identifier doi = MNodeService.getInstance(request).publish(session, metadataId); + Thread.sleep(30000); + System.out.println("+++++++++++++++++++ the metadataId on the ore package is "+metadataId.getValue()); + List oreIds = MNodeService.getInstance(request).lookupOreFor(session, doi, true); + assertTrue(oreIds.size() == 1); + List oreId2 = MNodeService.getInstance(request).lookupOreFor(session, dataId, true); + assertTrue(oreId2.size() == 2); } catch (Exception e) { e.printStackTrace(); fail("Unexpected error: " + e.getMessage()); } } - + /** * Test to publish a package */ From 1b092bf343b9d00e13e32e8aefe58322e084faa7 Mon Sep 17 00:00:00 2001 From: thomas Date: Thu, 22 Jul 2021 16:14:16 -0700 Subject: [PATCH 25/45] Add publishing test --- test/edu/ucsb/nceas/metacat/dataone/MNodeServiceTest.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/edu/ucsb/nceas/metacat/dataone/MNodeServiceTest.java b/test/edu/ucsb/nceas/metacat/dataone/MNodeServiceTest.java index 977f75a7d..a8e9ad5b3 100644 --- a/test/edu/ucsb/nceas/metacat/dataone/MNodeServiceTest.java +++ b/test/edu/ucsb/nceas/metacat/dataone/MNodeServiceTest.java @@ -1961,7 +1961,13 @@ public void testPublishPackage() { assertTrue(oreId3.size() == 1); //publish the package Identifier doi = MNodeService.getInstance(request).publish(session, metadataId); - + // test the ORE lookup + Thread.sleep(30000); + System.out.println("+++++++++++++++++++ the metadataId on the ore package is "+metadataId.getValue()); + List oreIds = MNodeService.getInstance(request).lookupOreFor(session, doi, true); + assertTrue(oreIds.size() == 1); + List oreId2 = MNodeService.getInstance(request).lookupOreFor(session, dataId, true); + assertTrue(oreId2.size() == 2); } catch (Exception e) { e.printStackTrace(); fail("Unexpected error: " + e.getMessage()); From f6745afcc13810b0b6b42e6c5fbde79cd410dd76 Mon Sep 17 00:00:00 2001 From: Jing Tao Date: Fri, 30 Jul 2021 17:04:46 -0700 Subject: [PATCH 26/45] Add the test class to test the multiple shoulder scenario. Now it only works on the ezid stage env. --- lib/metacat.properties | 6 +- .../dataone/MultipleDOIShouldersTest.java | 231 ++++++++++++++++++ .../metacat/dataone/RegisterDOITest.java | 6 +- 3 files changed, 237 insertions(+), 6 deletions(-) create mode 100644 test/edu/ucsb/nceas/metacat/dataone/MultipleDOIShouldersTest.java diff --git a/lib/metacat.properties b/lib/metacat.properties index 8e68b5934..d09cc0798 100755 --- a/lib/metacat.properties +++ b/lib/metacat.properties @@ -766,10 +766,10 @@ guid.ezid.baseurl=https://ezid.cdlib.org/ # optional path for target. do not include hostname here. is replaced with real value guid.ezid.uritemplate.metadata=/metacatui/view/ #guid.ezid.uritemplate.data= +#The first shoulder is the primary one. It is used for minting (generating) and creating (registering) DOIs. guid.ezid.doishoulder.1=doi:10.5072/FK2 -#guid.ezid.doishoulder.1=doi:10.5072/FK2/KNB/ -#guid.ezid.doishoulder.3=doi:10.5072/FK2/PISCO/ -#guid.ezid.doishoulder.6=doi:10.5072/FK2/LTER/ +#The second and beyond shoulders are only for creating (registering) DOIs. You need to generate a unique DOI by yourself. +guid.ezid.doishoulder.2= #The factory classes can generate the datacite document. It should be sperator by ';' if it has multiple values. guid.ezid.datacite.factories=edu.ucsb.nceas.metacat.doi.datacite.EML2DataCiteFactory diff --git a/test/edu/ucsb/nceas/metacat/dataone/MultipleDOIShouldersTest.java b/test/edu/ucsb/nceas/metacat/dataone/MultipleDOIShouldersTest.java new file mode 100644 index 000000000..6a4ef00f8 --- /dev/null +++ b/test/edu/ucsb/nceas/metacat/dataone/MultipleDOIShouldersTest.java @@ -0,0 +1,231 @@ +/** + * '$RCSfile$' + * Copyright: 2020 Regents of the University of California and the + * National Center for Ecological Analysis and Synthesis + * Purpose: To test the Access Controls in metacat by JUnit + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package edu.ucsb.nceas.metacat.dataone; + +import java.io.ByteArrayInputStream; +import java.io.FileInputStream; +import java.io.InputStream; +import java.util.HashMap; +import java.util.Set; +import java.util.UUID; + +import org.dataone.client.v2.formats.ObjectFormatCache; +import org.dataone.configuration.Settings; +import org.dataone.service.types.v1.Identifier; +import org.dataone.service.types.v1.Session; +import org.dataone.service.types.v2.SystemMetadata; +import org.junit.After; +import org.junit.Before; + +import edu.ucsb.nceas.ezid.EZIDService; +import edu.ucsb.nceas.ezid.profile.DataCiteProfile; +import edu.ucsb.nceas.ezid.profile.InternalProfile; +import edu.ucsb.nceas.metacat.properties.PropertyService; +import junit.framework.Test; +import junit.framework.TestSuite; + +/** + * To test the scenario that Metacat supports multiple shoulders. + * The first shoulder is the primary one. It uses for both minting and registering DOIs. + * The second and beyond shoulders are only for registering DOIs. + * Currently the testing only works on the ezid stage environment. + * @author tao + * + */ +public class MultipleDOIShouldersTest extends D1NodeServiceTest { + private static final String PROPERTY_SHOULDER_1 = "guid.ezid.doishoulder.1"; + private static final String PROPERTY_SHOULDER_2 = "guid.ezid.doishoulder.2"; + private static final String SHOULDER_1 = "doi:10.18739/A2"; + private static final String SHOULDER_2 = "doi:10.5063/"; + + /** + * Constructor + * @param name + */ + public MultipleDOIShouldersTest(String name) { + super(name); + } + + /** + * Set up the test fixtures + * @throws Exception + */ + @Before + public void setUp() throws Exception { + super.setUp(); + // set up the configuration for d1client + Settings.getConfiguration().setProperty("D1Client.cnClassName", + MockCNode.class.getName()); + PropertyService.getInstance().setPropertyNoPersist(PROPERTY_SHOULDER_1, SHOULDER_1); + PropertyService.getInstance().setPropertyNoPersist(PROPERTY_SHOULDER_2, SHOULDER_2); + } + + /** + * Remove the test fixtures + */ + @After + public void tearDown() { + } + + /** + * Build the test suite + * @return + */ + public static Test suite() { + TestSuite suite = new TestSuite(); + suite.addTest(new MultipleDOIShouldersTest("testPrimaryShoulder")); + suite.addTest(new MultipleDOIShouldersTest("testSecondaryShoulder")); + return suite; + } + + /** + * Test to mint and register an DOI by the primary (first) shoulder. + * @throws Exception + */ + public void testPrimaryShoulder() throws Exception { + printTestHeader("testPrimaryShoulder"); + try { + // get ezid config properties + String ezidUsername = PropertyService.getProperty("guid.ezid.username"); + String ezidPassword = PropertyService.getProperty("guid.ezid.password"); + String ezidServiceBaseUrl = PropertyService.getProperty("guid.ezid.baseurl"); + EZIDService ezid = new EZIDService(ezidServiceBaseUrl); + ezid.login(ezidUsername, ezidPassword); + + // Mint a DOI + Session session = getTestSession(); + Identifier guid = MNodeService.getInstance(request).generateIdentifier(session, "DOI", null); + assertTrue(guid.getValue().startsWith(SHOULDER_1)); + + // check that EZID knows about it + HashMap metadata = null; + int count = 0; + do { + try { + metadata = ezid.getMetadata(guid.getValue()); + } catch (Exception e) { + Thread.sleep(RegisterDOITest.SLEEP_TIME); + } + count++; + } while (metadata == null && count < RegisterDOITest.MAX_TIMES); + assertNotNull(metadata); + + // add the actual object for the newly-minted DOI + SystemMetadata sysmeta = null; + InputStream object = new FileInputStream(RegisterDOITest.EMLFILEPATH); + sysmeta = createSystemMetadata(guid, session.getSubject(), object); + object.close(); + object = new FileInputStream(RegisterDOITest.EMLFILEPATH); + sysmeta.setFormatId(ObjectFormatCache.getInstance().getFormat("eml://ecoinformatics.org/eml-2.1.0").getFormatId()); + Identifier pid = MNodeService.getInstance(request).create(session, guid, object, sysmeta); + assertEquals(guid.getValue(), pid.getValue()); + // check for the metadata for title element + count = 0; + metadata = null; + do { + try { + metadata = ezid.getMetadata(pid.getValue()); + // check if the update thread finished yet, otherwise try again + if (metadata != null) { + String registeredTarget = metadata.get(InternalProfile.TARGET.toString()); + if (!registeredTarget.contains(pid.getValue())) { + // try fetching it again + metadata = null; + } + } + } catch (Exception e) { + Thread.sleep(RegisterDOITest.SLEEP_TIME); + } + count++; + } while (metadata == null && count < RegisterDOITest.MAX_TIMES); + assertNotNull(metadata); + /*Set keys = metadata.keySet(); + for (String key : keys) { + System.out.println("=====the key " + key + " has the value of " + metadata.get(key)); + }*/ + assertTrue(metadata.containsKey(DataCiteProfile.TITLE.toString())); + + // check that the target URI was updated + String registeredTarget = metadata.get(InternalProfile.TARGET.toString()); + assertTrue(registeredTarget.contains(pid.getValue())); + } catch (Exception e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + } + } + + /** + * Test to register an DOI with the secondary shoulder. + * The DOI will be generated by ourself. + * @throws Exception + */ + public void testSecondaryShoulder() throws Exception { + printTestHeader("testSecondShoulder"); + UUID uuid = UUID.randomUUID(); + String uuidStr = uuid.toString(); + Identifier guid = new Identifier(); + guid.setValue(SHOULDER_2 + uuidStr); + System.out.println("the guid is " + guid.getValue()); + + Session session = getTestSession(); + SystemMetadata sysmeta = null; + InputStream object = new FileInputStream(RegisterDOITest.EMLFILEPATH); + sysmeta = createSystemMetadata(guid, session.getSubject(), object); + object.close(); + object = new FileInputStream(RegisterDOITest.EMLFILEPATH); + sysmeta.setFormatId(ObjectFormatCache.getInstance().getFormat("eml://ecoinformatics.org/eml-2.1.0").getFormatId()); + Identifier pid = MNodeService.getInstance(request).create(session, guid, object, sysmeta); + assertEquals(guid.getValue(), pid.getValue()); + // check for the metadata for title element + int count = 0; + HashMap metadata = null; + String ezidServiceBaseUrl = PropertyService.getProperty("guid.ezid.baseurl"); + EZIDService ezid = new EZIDService(ezidServiceBaseUrl); + do { + try { + metadata = ezid.getMetadata(pid.getValue()); + // check if the update thread finished yet, otherwise try again + if (metadata != null) { + String registeredTarget = metadata.get(InternalProfile.TARGET.toString()); + if (!registeredTarget.contains(pid.getValue())) { + // try fetching it again + metadata = null; + } + } + } catch (Exception e) { + Thread.sleep(RegisterDOITest.SLEEP_TIME); + } + count++; + } while (metadata == null && count < RegisterDOITest.MAX_TIMES); + assertNotNull(metadata); + /*Set keys = metadata.keySet(); + for (String key : keys) { + System.out.println("=====the key " + key + " has the value of " + metadata.get(key)); + }*/ + assertTrue(metadata.containsKey("datacite")); + String datacite = metadata.get("datacite"); + assertTrue(datacite.contains("Test EML package - public-readable from morpho")); + // check that the target URI was updated + String registeredTarget = metadata.get(InternalProfile.TARGET.toString()); + assertTrue(registeredTarget.contains(pid.getValue())); + } + +} diff --git a/test/edu/ucsb/nceas/metacat/dataone/RegisterDOITest.java b/test/edu/ucsb/nceas/metacat/dataone/RegisterDOITest.java index 04dc8bc51..802acf44f 100644 --- a/test/edu/ucsb/nceas/metacat/dataone/RegisterDOITest.java +++ b/test/edu/ucsb/nceas/metacat/dataone/RegisterDOITest.java @@ -66,9 +66,9 @@ * */ public class RegisterDOITest extends D1NodeServiceTest { - private static final int MAX_TIMES = 50; - private static final int SLEEP_TIME = 3000; - private static final String EMLFILEPATH = "test/tao.14563.1.xml"; + public static final int MAX_TIMES = 50; + public static final int SLEEP_TIME = 3000; + public static final String EMLFILEPATH = "test/tao.14563.1.xml"; public static final String creatorsStr = "onlySurNameNational Center for Ecological Analysis and SynthesisSmith, JohnKing, WendyUniversity of California Santa Barbara"; /** From 0652747ca4facb37a50f1d2225d60535df4f18b1 Mon Sep 17 00:00:00 2001 From: Jing Tao Date: Mon, 2 Aug 2021 11:32:31 -0700 Subject: [PATCH 27/45] Add a new test case to test an DOI which is not in the configuration file. --- .../dataone/MultipleDOIShouldersTest.java | 56 +++++++++++++++++-- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/test/edu/ucsb/nceas/metacat/dataone/MultipleDOIShouldersTest.java b/test/edu/ucsb/nceas/metacat/dataone/MultipleDOIShouldersTest.java index 6a4ef00f8..df21342d6 100644 --- a/test/edu/ucsb/nceas/metacat/dataone/MultipleDOIShouldersTest.java +++ b/test/edu/ucsb/nceas/metacat/dataone/MultipleDOIShouldersTest.java @@ -93,6 +93,7 @@ public static Test suite() { TestSuite suite = new TestSuite(); suite.addTest(new MultipleDOIShouldersTest("testPrimaryShoulder")); suite.addTest(new MultipleDOIShouldersTest("testSecondaryShoulder")); + suite.addTest(new MultipleDOIShouldersTest("testNonExistedShoulder")); return suite; } @@ -103,13 +104,9 @@ public static Test suite() { public void testPrimaryShoulder() throws Exception { printTestHeader("testPrimaryShoulder"); try { - // get ezid config properties - String ezidUsername = PropertyService.getProperty("guid.ezid.username"); - String ezidPassword = PropertyService.getProperty("guid.ezid.password"); String ezidServiceBaseUrl = PropertyService.getProperty("guid.ezid.baseurl"); EZIDService ezid = new EZIDService(ezidServiceBaseUrl); - ezid.login(ezidUsername, ezidPassword); - + // Mint a DOI Session session = getTestSession(); Identifier guid = MNodeService.getInstance(request).generateIdentifier(session, "DOI", null); @@ -227,5 +224,54 @@ public void testSecondaryShoulder() throws Exception { String registeredTarget = metadata.get(InternalProfile.TARGET.toString()); assertTrue(registeredTarget.contains(pid.getValue())); } + + /** + * Test to register an DOI with a shoulder which is not in the configuration file. + * The DOI will be generated by ourself. + * @throws Exception + */ + public void testNonExistedShoulder() throws Exception { + printTestHeader("testNonExistedShoulder"); + UUID shoulderUuid = UUID.randomUUID(); + String shoulder = "doi:99.000/" + shoulderUuid.toString() + "/"; + + UUID uuid = UUID.randomUUID(); + String uuidStr = uuid.toString(); + Identifier guid = new Identifier(); + guid.setValue(shoulder + uuidStr); + System.out.println("the guid is " + guid.getValue()); + + Session session = getTestSession(); + SystemMetadata sysmeta = null; + InputStream object = new FileInputStream(RegisterDOITest.EMLFILEPATH); + sysmeta = createSystemMetadata(guid, session.getSubject(), object); + object.close(); + object = new FileInputStream(RegisterDOITest.EMLFILEPATH); + sysmeta.setFormatId(ObjectFormatCache.getInstance().getFormat("eml://ecoinformatics.org/eml-2.1.0").getFormatId()); + Identifier pid = MNodeService.getInstance(request).create(session, guid, object, sysmeta); + assertEquals(guid.getValue(), pid.getValue()); + // check for the metadata for title element + int count = 0; + HashMap metadata = null; + String ezidServiceBaseUrl = PropertyService.getProperty("guid.ezid.baseurl"); + EZIDService ezid = new EZIDService(ezidServiceBaseUrl); + do { + try { + metadata = ezid.getMetadata(pid.getValue()); + // check if the update thread finished yet, otherwise try again + if (metadata != null) { + String registeredTarget = metadata.get(InternalProfile.TARGET.toString()); + if (!registeredTarget.contains(pid.getValue())) { + // try fetching it again + metadata = null; + } + } + } catch (Exception e) { + Thread.sleep(RegisterDOITest.SLEEP_TIME); + } + count++; + } while (metadata == null && count < RegisterDOITest.MAX_TIMES); + assertNull(metadata); + } } From 6b4c51eb9729f4bcf300448da7d598f5e3d700c5 Mon Sep 17 00:00:00 2001 From: Bryce Date: Fri, 23 Jul 2021 17:00:12 -0800 Subject: [PATCH 28/45] Refactor EML semantics XSL to support separate popovers This change depends on MetacatUI! But I've made it so it's mostly backward compatible. With older versions of MetacatUI, you get the old behavior (one popover per annotation) with just a minor graphical difference. With newer MetacatUI builds, you'll get separate property and value popovers with no graphical glitches. Ref https://github.com/NCEAS/metacatui/issues/1807 --- .../skins/metacatui/eml-2/eml-semantics.xsl | 32 +++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/lib/style/skins/metacatui/eml-2/eml-semantics.xsl b/lib/style/skins/metacatui/eml-2/eml-semantics.xsl index 08573e4b0..410d8d515 100644 --- a/lib/style/skins/metacatui/eml-2/eml-semantics.xsl +++ b/lib/style/skins/metacatui/eml-2/eml-semantics.xsl @@ -39,8 +39,9 @@ - + annotation true @@ -51,15 +52,26 @@ -
-
- - annotation-findmore tooltip-this - tooltip - bottom - -
- +
+ + + + bottom + true + + +
+
+ + + + bottom + true + +
+ +
+
From fc46bfdcbe1453c5512b0b708124f476e48024fe Mon Sep 17 00:00:00 2001 From: Bryce Date: Tue, 27 Jul 2021 15:18:18 -0800 Subject: [PATCH 29/45] Extend semantic annotation indexing to include properties Closes https://github.com/NCEAS/metacat/issues/1517 This change requires a reindex of any content that has semantic annotations A similar change will go in https://github.com/DataONEorg/d1_cn_index_processor/issues/27 too --- .../application-context-eml-annotation.xml | 2 +- ...ication-context-ontology-model-service.xml | 29 +++++++++++++++---- .../EmlAnnotationSubprocessorTest.java | 5 ++++ .../annotation/OntologyModelServiceTest.java | 2 +- .../test/resources/eml-annotation-example.xml | 6 +++- 5 files changed, 36 insertions(+), 8 deletions(-) diff --git a/metacat-index/src/main/resources/application-context-eml-annotation.xml b/metacat-index/src/main/resources/application-context-eml-annotation.xml index 26f4a1092..b8a3cdde6 100644 --- a/metacat-index/src/main/resources/application-context-eml-annotation.xml +++ b/metacat-index/src/main/resources/application-context-eml-annotation.xml @@ -17,7 +17,7 @@ - + diff --git a/metacat-index/src/main/resources/application-context-ontology-model-service.xml b/metacat-index/src/main/resources/application-context-ontology-model-service.xml index f9e1c11dd..cb11550f1 100644 --- a/metacat-index/src/main/resources/application-context-ontology-model-service.xml +++ b/metacat-index/src/main/resources/application-context-ontology-model-service.xml @@ -6,7 +6,8 @@ - + + @@ -57,8 +58,8 @@ - - + + PREFIX owl: - SELECT ?sem_annotation + SELECT ?annotation_property_uri WHERE { - <$CONCEPT_URI> rdfs:subClassOf* ?sem_annotation . + <$CONCEPT_URI> rdfs:subPropertyOf* ?annotation_property_uri . + } + ]]> + + + + + + + + + + PREFIX rdfs: + PREFIX owl: + + SELECT ?annotation_value_uri + WHERE { + <$CONCEPT_URI> rdfs:subClassOf* ?annotation_value_uri . } ]]> diff --git a/metacat-index/src/test/java/edu/ucsb/nceas/metacat/index/annotation/EmlAnnotationSubprocessorTest.java b/metacat-index/src/test/java/edu/ucsb/nceas/metacat/index/annotation/EmlAnnotationSubprocessorTest.java index c6d43affd..c5057d6d4 100644 --- a/metacat-index/src/test/java/edu/ucsb/nceas/metacat/index/annotation/EmlAnnotationSubprocessorTest.java +++ b/metacat-index/src/test/java/edu/ucsb/nceas/metacat/index/annotation/EmlAnnotationSubprocessorTest.java @@ -67,6 +67,11 @@ public void testProcessDocument() throws Exception { assertTrue(values.contains("http://purl.dataone.org/odo/ECSO_00001105")); assertTrue(values.contains("http://purl.dataone.org/odo/ECSO_00000531")); assertTrue(values.contains("http://purl.dataone.org/odo/ECSO_00001143")); + assertTrue(values.contains("http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#containsMeasurementsOfType")); + assertTrue(values.contains("http://purl.obolibrary.org/obo/RO_0002352")); + assertTrue(values.contains("http://purl.obolibrary.org/obo/RO_0000056")); + assertTrue(values.contains("http://purl.obolibrary.org/obo/RO_0002328")); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); resultSolrDoc.serialize(baos, "UTF-8"); diff --git a/metacat-index/src/test/java/edu/ucsb/nceas/metacat/index/annotation/OntologyModelServiceTest.java b/metacat-index/src/test/java/edu/ucsb/nceas/metacat/index/annotation/OntologyModelServiceTest.java index 300186196..47e1d8cf6 100644 --- a/metacat-index/src/test/java/edu/ucsb/nceas/metacat/index/annotation/OntologyModelServiceTest.java +++ b/metacat-index/src/test/java/edu/ucsb/nceas/metacat/index/annotation/OntologyModelServiceTest.java @@ -54,7 +54,7 @@ public void testOntologyModelService() throws Exception { System.out.println("FOOBAR: " + service.getClass().getName()); List fieldList = service.getFieldList(); - assertEquals(1, fieldList.size()); + assertEquals(2, fieldList.size()); Map altEntries = service.getAltEntryList(); assertEquals(16, altEntries.size()); diff --git a/metacat-index/src/test/resources/eml-annotation-example.xml b/metacat-index/src/test/resources/eml-annotation-example.xml index 50831a92b..3601a7f90 100644 --- a/metacat-index/src/test/resources/eml-annotation-example.xml +++ b/metacat-index/src/test/resources/eml-annotation-example.xml @@ -16,9 +16,13 @@ http://purl.dataone.org/odo/ECSO_00000536 - http://www.w3.org/1999/02/22-rdf-syntax-ns#type + http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#containsMeasurementsOfType http://purl.dataone.org/odo/ECSO_00000531 + + http://purl.obolibrary.org/obo/RO_0002352 + http://purl.dataone.org/odo/ECSO_00000531 + test.user From d94a6a37d424f183ccc0a370a70245fa1dce755f Mon Sep 17 00:00:00 2001 From: Jing Tao Date: Thu, 19 Aug 2021 11:57:37 -0700 Subject: [PATCH 30/45] Add or move the statement to close the ldap context into the try-finally block. Ref:https://github.com/NCEAS/metacat/issues/1524 --- src/edu/ucsb/nceas/metacat/AuthLdap.java | 258 ++++++++++++----------- 1 file changed, 138 insertions(+), 120 deletions(-) diff --git a/src/edu/ucsb/nceas/metacat/AuthLdap.java b/src/edu/ucsb/nceas/metacat/AuthLdap.java index 15a0bcc2b..8390095a4 100755 --- a/src/edu/ucsb/nceas/metacat/AuthLdap.java +++ b/src/edu/ucsb/nceas/metacat/AuthLdap.java @@ -80,7 +80,7 @@ public class AuthLdap implements AuthInterface { private int ldapSearchCountLimit; private String currentReferralInfo; Hashtable env = new Hashtable(11); - private Context rContext; + //private Context rContext; private String userName; private String userPassword; ReferralException refExc; @@ -363,30 +363,37 @@ private String getAliasedDn(String alias, Hashtable env, boolean if(env != null) { env.put(Context.REFERRAL, "ignore"); } - LdapContext sctx = new InitialLdapContext(env, null); + LdapContext sctx = null; StartTlsResponse tls = null; - if(useTLS) { - tls = (StartTlsResponse) sctx.extendedOperation(new StartTlsRequest()); - // Open a TLS connection (over the existing LDAP association) and get details - // of the negotiated TLS session: cipher suite, peer certificate, etc. - SSLSession session = tls.negotiate(); - } - SearchControls ctls = new SearchControls(); - ctls.setSearchScope(SearchControls.SUBTREE_SCOPE); - String filter = "(objectClass=*)"; - NamingEnumeration answer = sctx.search(alias, filter, ctls); - while(answer.hasMore()) { - SearchResult result = (SearchResult) answer.next(); - if(!result.isRelative()) { - //if is not relative, this will be alias. - aliasedDn = result.getNameInNamespace(); - break; + try { + sctx = new InitialLdapContext(env, null); + if(useTLS) { + tls = (StartTlsResponse) sctx.extendedOperation(new StartTlsRequest()); + // Open a TLS connection (over the existing LDAP association) and get details + // of the negotiated TLS session: cipher suite, peer certificate, etc. + SSLSession session = tls.negotiate(); + } + SearchControls ctls = new SearchControls(); + ctls.setSearchScope(SearchControls.SUBTREE_SCOPE); + String filter = "(objectClass=*)"; + NamingEnumeration answer = sctx.search(alias, filter, ctls); + while(answer.hasMore()) { + SearchResult result = (SearchResult) answer.next(); + if(!result.isRelative()) { + //if is not relative, this will be alias. + aliasedDn = result.getNameInNamespace(); + break; + } + } + + } finally { + if(useTLS && tls != null) { + tls.close(); + } + if (sctx != null) { + sctx.close(); } } - if(useTLS && tls != null) { - tls.close(); - } - sctx.close(); return aliasedDn; } @@ -394,15 +401,16 @@ private String getAliasedDn(String alias, Hashtable env, boolean private boolean authenticateTLS(Hashtable env, String userDN, String password) throws AuthTLSException, AuthenticationException{ logMetacat.info("AuthLdap.authenticateTLS - Trying to authenticate with TLS"); + LdapContext ctx = null; + StartTlsResponse tls = null; try { - LdapContext ctx = null; double startTime; double stopTime; startTime = System.currentTimeMillis(); ctx = new InitialLdapContext(env, null); // Start up TLS here so that we don't pass our jewels in // cleartext - StartTlsResponse tls = + tls = (StartTlsResponse) ctx.extendedOperation(new StartTlsRequest()); // tls.setHostnameVerifier(new SampleVerifier()); SSLSession sess = tls.negotiate(); @@ -421,6 +429,21 @@ private boolean authenticateTLS(Hashtable env, String userDN, St throw new AuthTLSException("AuthLdap.authenticateTLS - Naming error when athenticating via TLS: " + ne.getMessage()); } catch (IOException ioe) { throw new AuthTLSException("AuthLdap.authenticateTLS - I/O error when athenticating via TLS: " + ioe.getMessage()); + } finally { + if (tls != null) { + try { + tls.close(); + } catch (IOException ee) { + logMetacat.error("AuthLdap.authenticateTLS - can't close the TlsResponse since " + ee.getMessage()); + } + } + if (ctx != null) { + try { + ctx.close(); + } catch (NamingException ee) { + logMetacat.error("AuthLdap.authenticateTLS - can't close the LdapContext since " + ee.getMessage()); + } + } } return true; } @@ -428,24 +451,29 @@ private boolean authenticateTLS(Hashtable env, String userDN, St private boolean authenticateNonTLS(Hashtable env, String userDN, String password) throws NamingException { LdapContext ctx = null; - double startTime; - double stopTime; - - logMetacat.info("AuthLdap.authenticateNonTLS - Trying to authenticate without TLS"); - //env.put(Context.SECURITY_AUTHENTICATION, "simple"); - //env.put(Context.SECURITY_PRINCIPAL, userDN); - //env.put(Context.SECURITY_CREDENTIALS, password); - - startTime = System.currentTimeMillis(); - ctx = new InitialLdapContext(env, null); - ctx.addToEnvironment(Context.SECURITY_AUTHENTICATION, "simple"); - ctx.addToEnvironment(Context.SECURITY_PRINCIPAL, userDN); - ctx.addToEnvironment(Context.SECURITY_CREDENTIALS, password); - ctx.reconnect(null); - stopTime = System.currentTimeMillis(); - logMetacat.info("AuthLdap.authenticateNonTLS - Connection time thru " + ldapsUrl + " was: " - + (stopTime - startTime) / 1000 + " seconds."); - + try { + double startTime; + double stopTime; + + logMetacat.info("AuthLdap.authenticateNonTLS - Trying to authenticate without TLS"); + //env.put(Context.SECURITY_AUTHENTICATION, "simple"); + //env.put(Context.SECURITY_PRINCIPAL, userDN); + //env.put(Context.SECURITY_CREDENTIALS, password); + + startTime = System.currentTimeMillis(); + ctx = new InitialLdapContext(env, null); + ctx.addToEnvironment(Context.SECURITY_AUTHENTICATION, "simple"); + ctx.addToEnvironment(Context.SECURITY_PRINCIPAL, userDN); + ctx.addToEnvironment(Context.SECURITY_CREDENTIALS, password); + ctx.reconnect(null); + stopTime = System.currentTimeMillis(); + logMetacat.info("AuthLdap.authenticateNonTLS - Connection time thru " + ldapsUrl + " was: " + + (stopTime - startTime) / 1000 + " seconds."); + } finally { + if (ctx != null) { + ctx.close(); + } + } return true; } @@ -466,6 +494,7 @@ private String getIdentifyingName(String user, String ldapUrl, String ldapBase) env.put(Context.INITIAL_CONTEXT_FACTORY, "com.sun.jndi.ldap.LdapCtxFactory"); env.put(Context.REFERRAL, "throw"); env.put(Context.PROVIDER_URL, ldapUrl + ldapBase); + DirContext sctx = null; try { int position = user.indexOf(","); String uid = user.substring(user.indexOf("=") + 1, position); @@ -474,7 +503,7 @@ private String getIdentifyingName(String user, String ldapUrl, String ldapBase) .indexOf(",", position + 1)); logMetacat.info("AuthLdap.getIdentifyingName - org is: " + org); - DirContext sctx = new InitialDirContext(env); + sctx = new InitialDirContext(env); SearchControls ctls = new SearchControls(); ctls.setSearchScope(SearchControls.SUBTREE_SCOPE); String filter = "(&(uid=" + uid + ")(o=" + org + "))"; @@ -518,6 +547,10 @@ private String getIdentifyingName(String user, String ldapUrl, String ldapBase) logMetacat.error("AuthLdap.getIdentifyingName - Naming exception while getting dn: " + e); throw new NamingException("Naming exception in AuthLdap.getIdentifyingName: " + e); + } finally { + if (sctx != null) { + sctx.close(); + } } return identifier; } @@ -540,11 +573,11 @@ public String[][] getUsers(String user, String password) throws ConnectException env.put(Context.REFERRAL, referral); env.put(Context.PROVIDER_URL, ldapUrl); env.put("com.sun.jndi.ldap.connect.timeout", ldapConnectTimeLimit); - + DirContext ctx = null; try { // Create the initial directory context - DirContext ctx = new InitialDirContext(env); + ctx = new InitialDirContext(env); // Specify the attributes to match. // Users are objects that have the attribute @@ -610,10 +643,6 @@ public String[][] getUsers(String user, String password) throws ConnectException users[i][3] = (String) uorg.elementAt(i); users[i][4] = (String) umail.elementAt(i); } - - // Close the context when we're done - ctx.close(); - } catch (NamingException e) { logMetacat.error("AuthLdap.getUsers - Problem getting users in AuthLdap.getUsers:" + e); // e.printStackTrace(System.err); @@ -621,8 +650,16 @@ public String[][] getUsers(String user, String password) throws ConnectException * throw new ConnectException( "Problem getting users in * AuthLdap.getUsers:" + e); */ + } finally { + // Close the context when we're done + try { + if (ctx != null ) { + ctx.close(); + } + } catch (NamingException ee) { + logMetacat.error("AuthLdap.getUsers - can't close the LdapContext since " + ee.getMessage()); + } } - return users; } @@ -654,12 +691,12 @@ public String[] getUserInfo(String user, String password) throws ConnectExceptio //the the user is an alias name. we need to use the the real name user = realName; } - + DirContext ctx = null; try { // Create the initial directory context env.put(Context.REFERRAL, referral); - DirContext ctx = new InitialDirContext(env); + ctx = new InitialDirContext(env); // Specify the attributes to match. // Users are objects that have the attribute // objectclass=InetOrgPerson. @@ -711,16 +748,20 @@ public String[] getUserInfo(String user, String password) throws ConnectExceptio logMetacat.error("AuthLdap.getUserInfo - LDAP Server size limit exceeded. " + "Returning incomplete record set."); } - - // Close the context when we're done - ctx.close(); - } catch (NamingException e) { logMetacat.error("AuthLdap.getUserInfo - Problem getting users:" + e); // e.printStackTrace(System.err); throw new ConnectException("Problem getting users in AuthLdap.getUsers:" + e); + } finally { + // Close the context when we're done + if (ctx != null) { + try { + ctx.close(); + } catch (NamingException ee) { + logMetacat.error("AuthLdap.getUserInfo - can't close the LdapContext since " + ee.getMessage()); + } + } } - return userinfo; } @@ -744,11 +785,11 @@ public String[] getUsers(String user, String password, String group) env.put(Context.INITIAL_CONTEXT_FACTORY, "com.sun.jndi.ldap.LdapCtxFactory"); env.put(Context.REFERRAL, referral); env.put(Context.PROVIDER_URL, ldapUrl); - + DirContext ctx = null; try { // Create the initial directory context - DirContext ctx = new InitialDirContext(env); + ctx = new InitialDirContext(env); // Specify the ids of the attributes to return String[] attrIDs = { "uniqueMember" }; @@ -774,10 +815,6 @@ public String[] getUsers(String user, String password, String group) for (int i = 0; i < uvec.size(); i++) { users[i] = (String) uvec.elementAt(i); } - - // Close the context when we're done - ctx.close(); - } catch (NamingException e) { logMetacat.error("AuthLdap.getUsers - Problem getting users for a group in " + "AuthLdap.getUsers:" + e); @@ -785,6 +822,15 @@ public String[] getUsers(String user, String password, String group) * throw new ConnectException( "Problem getting users for a group in * AuthLdap.getUsers:" + e); */ + } finally { + // Close the context when we're done + if (ctx != null) { + try { + ctx.close(); + } catch (NamingException ee) { + logMetacat.error("AuthLdap.getUsers - can't close the LdapContext since " + ee.getMessage()); + } + } } return users; @@ -849,10 +895,11 @@ public String[][] getGroups(String user, String password, String foruser) // Iterate through the referrals, handling NamingExceptions in the // outer catch statement, ReferralExceptions in the inner catch // statement + DirContext ctx = null; try { // outer try // Create the initial directory context - DirContext ctx = new InitialDirContext(env); + ctx = new InitialDirContext(env); // Specify the attributes to match. // Groups are objects with attribute objectclass=groupofuniquenames. @@ -1025,11 +1072,6 @@ public String[][] getGroups(String user, String password, String foruser) }// end inner try }// end for - - // close the context now that all initial and referral - // searches are processed - ctx.close(); - } catch (NamingException e) { // naming exceptions get logged, groups are returned @@ -1037,6 +1079,13 @@ public String[][] getGroups(String user, String password, String foruser) e.printStackTrace(System.err); } finally { + if (ctx != null) { + try { + ctx.close(); + } catch (NamingException ee) { + logMetacat.error("AuthLdap.getGroups - can't close the LdapContext since " + ee.getMessage()); + } + } // once all referrals are followed, report and return the groups // found logMetacat.warn("AuthLdap.getGroups - The user is in the following groups: " + gvec.toString()); @@ -1085,11 +1134,11 @@ public HashMap> getAttributes(String user, String passwor env.put(Context.INITIAL_CONTEXT_FACTORY, "com.sun.jndi.ldap.LdapCtxFactory"); env.put(Context.REFERRAL, referral); env.put(Context.PROVIDER_URL, ldapUrl); - + DirContext ctx = null; try { // Create the initial directory context - DirContext ctx = new InitialDirContext(env); + ctx = new InitialDirContext(env); // Ask for all attributes of the user // Attributes attrs = ctx.getAttributes(userident); @@ -1113,14 +1162,19 @@ public HashMap> getAttributes(String user, String passwor } attributes.put(attName, values); } - - // Close the context when we're done - ctx.close(); } catch (NamingException e) { logMetacat.error("AuthLdap.getAttributes - Problem getting attributes:" + e); throw new ConnectException( "Problem getting attributes in AuthLdap.getAttributes:" + e); + } finally { + if (ctx != null) { + try { + ctx.close(); + } catch (NamingException ee) { + logMetacat.error("AuthLdap.getAttributes - can't close the LdapContext since " + ee.getMessage()); + } + } } return attributes; @@ -1147,11 +1201,11 @@ private Hashtable getSubtrees(String user, String password, String ldapUrl, env.put(Context.REFERRAL, "ignore"); env.put(Context.PROVIDER_URL, ldapUrl + ldapBase); - + DirContext ctx = null; try { // Create the initial directory context - DirContext ctx = new InitialDirContext(env); + ctx = new InitialDirContext(env); // Specify the ids of the attributes to return String[] attrIDs = { "o", "ref" }; @@ -1221,14 +1275,18 @@ private Hashtable getSubtrees(String user, String password, String ldapUrl, } } } - - // Close the context when we're done - ctx.close(); - } catch (NamingException e) { logMetacat.error("AuthLdap.getSubtrees - Problem getting subtrees in AuthLdap.getSubtrees:" + e); throw new ConnectException( "Problem getting subtrees in AuthLdap.getSubtrees:" + e); + } finally { + if (ctx != null) { + try { + ctx.close(); + } catch (NamingException ee) { + logMetacat.error("AuthLdap.getSubtrees - can't close the LdapContext since " + ee.getMessage()); + } + } } return trees; @@ -1371,46 +1429,6 @@ public static int searchUser(String user, String userGroup[][]) { return -1; } - public void testCredentials(String dn, String password, String rootServer, - String rootBase) throws NamingException { - - String server = ""; - String userDN = ""; - logMetacat.debug("dn is: " + dn); - - int position = dn.lastIndexOf("/"); - logMetacat.debug("AuthLdap.testCredentials - position is: " + position); - if (position == -1) { - server = rootServer; - if (dn.indexOf(userDN) < 0) { - userDN = dn + "," + rootBase; - } else { - userDN = dn; - } - logMetacat.debug("AuthLdap.testCredentials - userDN is: " + userDN); - - } else { - server = dn.substring(0, position + 1); - userDN = dn.substring(position + 1); - logMetacat.debug("AuthLdap.testCredentials - server is: " + server); - logMetacat.debug("AuthLdap.testCredentials - userDN is: " + userDN); - } - - logMetacat.debug("AuthLdap.testCredentials - Trying to authenticate: " + userDN + " using server: " + server); - - // /* try { - LdapContext ctx = null; - - env.put(Context.INITIAL_CONTEXT_FACTORY, "com.sun.jndi.ldap.LdapCtxFactory"); - env.put(Context.REFERRAL, "follow"); - env.put(Context.SECURITY_AUTHENTICATION, "simple"); - env.put(Context.SECURITY_PRINCIPAL, userDN); - env.put(Context.SECURITY_CREDENTIALS, password); - env.put(Context.PROVIDER_URL, rootServer); - - ctx = new InitialLdapContext(env, null); - - } /** * Test method for the class From 7ed62b7cc1b5f68c28fbea415088e7a6ebd21280 Mon Sep 17 00:00:00 2001 From: Jing Tao Date: Wed, 25 Aug 2021 10:40:57 -0700 Subject: [PATCH 31/45] Make the AuthSession instance from a local variable to a static class field. So we can decrease the number of objects in JVM. --- .../restservice/D1ResourceHandler.java | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/edu/ucsb/nceas/metacat/restservice/D1ResourceHandler.java b/src/edu/ucsb/nceas/metacat/restservice/D1ResourceHandler.java index a56be726d..a3cb1e48a 100644 --- a/src/edu/ucsb/nceas/metacat/restservice/D1ResourceHandler.java +++ b/src/edu/ucsb/nceas/metacat/restservice/D1ResourceHandler.java @@ -113,6 +113,8 @@ public class D1ResourceHandler { protected static final String FUNCTION_NAME_INSERT = "insert"; protected static final String FUNCTION_NAME_UPDATE = "update"; + protected static AuthSession auth = null; + protected ServletContext servletContext; protected static Log logMetacat; protected HttpServletRequest request; @@ -198,11 +200,21 @@ public void handle(byte httpVerb) { String password = null; String[] groups = null; - try { - AuthSession auth = new AuthSession(); - groups = auth.getGroups(username, password, dn); - } catch (Exception e) { - logMetacat.warn("D1ReourceHandler.handle - we can't get group information for the user "+dn+" from the authentication interface since :", e); + if (auth == null) { + try { + auth = new AuthSession(); + groups = auth.getGroups(username, password, dn); + } catch (Exception e) { + logMetacat.warn("D1ReourceHandler.handle - we can't get group information for the user " + dn + + " from the authentication interface since :", e); + } + } else { + try { + groups = auth.getGroups(username, password, dn); + } catch (Exception e) { + logMetacat.warn("D1ReourceHandler.handle - we can't get group information for the user " + dn + + " from the authentication interface since :", e); + } } if(groups != null) { From 6b2d3d512766f67444dbef2707e4b8b1412eb3e2 Mon Sep 17 00:00:00 2001 From: Jing Tao Date: Thu, 26 Aug 2021 16:44:00 -0700 Subject: [PATCH 32/45] Add a property to control if we need to append ldap accounts to a session. --- lib/metacat.properties | 3 + .../restservice/D1ResourceHandler.java | 164 ++++++++++-------- 2 files changed, 91 insertions(+), 76 deletions(-) diff --git a/lib/metacat.properties b/lib/metacat.properties index d09cc0798..02c703576 100755 --- a/lib/metacat.properties +++ b/lib/metacat.properties @@ -756,6 +756,9 @@ dataone.quotas.dailyReportingUsagesTime=11:00 PM # use ; to separate multiple name spaces. dataone.quotas.portal.namespaces=https://purl.dataone.org/portals-1.0.0;https://purl.dataone.org/portals-1.1.0 +# Enable Metacat to append ldap groups to a session +dataone.session.appendLdapGroups.enabled=true + ############# Global Identifiers Assignment Section ###################### guid.assignGUIDs=false guid.ezid.enabled=false diff --git a/src/edu/ucsb/nceas/metacat/restservice/D1ResourceHandler.java b/src/edu/ucsb/nceas/metacat/restservice/D1ResourceHandler.java index a3cb1e48a..b195451ef 100644 --- a/src/edu/ucsb/nceas/metacat/restservice/D1ResourceHandler.java +++ b/src/edu/ucsb/nceas/metacat/restservice/D1ResourceHandler.java @@ -46,6 +46,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.dataone.client.v2.itk.D1Client; +import org.dataone.configuration.Settings; import org.dataone.exceptions.MarshallingException; import org.dataone.mimemultipart.MultipartRequest; import org.dataone.mimemultipart.MultipartRequestResolver; @@ -115,6 +116,8 @@ public class D1ResourceHandler { protected static AuthSession auth = null; + protected static boolean enableAppendLdapGroups = Settings.getConfiguration().getBoolean("dataone.session.appendLdapGroups.enabled", true); + protected ServletContext servletContext; protected static Log logMetacat; protected HttpServletRequest request; @@ -189,90 +192,99 @@ public void handle(byte httpVerb) { } } } else { - //The session is not null. However, the if we got the session is from a token, the ldap group information for is missing if we logged in by the ldap account. - //here we just patch it. - Subject subject = session.getSubject(); - if(subject != null) { - String dn = subject.getValue(); - logMetacat.debug("D1ReourceHandler.handle - the subject dn in the session is "+dn+" This dn will be used to look up the group information"); - if(dn != null) { - String username = null; - String password = null; - - String[] groups = null; - if (auth == null) { - try { - auth = new AuthSession(); - groups = auth.getGroups(username, password, dn); - } catch (Exception e) { - logMetacat.warn("D1ReourceHandler.handle - we can't get group information for the user " + dn + - " from the authentication interface since :", e); - } - } else { - try { - groups = auth.getGroups(username, password, dn); - } catch (Exception e) { - logMetacat.warn("D1ReourceHandler.handle - we can't get group information for the user " + dn + - " from the authentication interface since :", e); - } - } - - if(groups != null) { - SubjectInfo subjectInfo = session.getSubjectInfo(); - if(subjectInfo != null) { - logMetacat.debug("D1ReourceHandler.handle - the subject information is NOT null when we try to figure out the group information."); - //we don't overwrite the existing subject info, just add the new groups informations - List persons = subjectInfo.getPersonList(); - Person targetPerson = null; - if(persons != null) { - for(Person person : persons) { - if(person.getSubject().equals(subject)) { - targetPerson = person; - logMetacat.debug("D1ReourceHandler.handle - we find a person with the subject "+dn+" in the subject info."); - break; + //The session is not null. However, if we got the session is from a token, the local ldap group information is missing when we logged in by the ldap account. + //Here we just patch it (d1_portal only patches the dataone groups) + if (enableAppendLdapGroups) { + logMetacat.debug("D1ReourceHandler.handle - Metacat is configured to append the local ldap group information to a session."); + Subject subject = session.getSubject(); + if(subject != null) { + String dn = subject.getValue(); + logMetacat.debug("D1ReourceHandler.handle - the subject dn in the session is "+dn+" This dn will be used to look up the group information"); + if(dn != null) { + String username = null; + String password = null; + String[] groups = null; + if (auth == null) { + try { + synchronized (D1ResourceHandler.class) { + if (auth == null) { + auth = new AuthSession(); } } - } - boolean newPerson = false; - if(targetPerson == null) { - newPerson = true; - targetPerson = new Person(); - targetPerson.setSubject(subject); - } - for (int i=0; i persons = subjectInfo.getPersonList(); + Person targetPerson = null; + if(persons != null) { + for(Person person : persons) { + if(person.getSubject().equals(subject)) { + targetPerson = person; + logMetacat.debug("D1ReourceHandler.handle - we find a person with the subject "+dn+" in the subject info."); + break; + } + } + } + boolean newPerson = false; + if(targetPerson == null) { + newPerson = true; + targetPerson = new Person(); + targetPerson.setSubject(subject); + } + for (int i=0; i Date: Mon, 30 Aug 2021 08:57:32 -0700 Subject: [PATCH 33/45] Add a LRUMap object to cache the group information. --- src/edu/ucsb/nceas/metacat/AuthSession.java | 61 +++++++++++++++++---- 1 file changed, 51 insertions(+), 10 deletions(-) diff --git a/src/edu/ucsb/nceas/metacat/AuthSession.java b/src/edu/ucsb/nceas/metacat/AuthSession.java index be31534d2..9b9ebfd94 100755 --- a/src/edu/ucsb/nceas/metacat/AuthSession.java +++ b/src/edu/ucsb/nceas/metacat/AuthSession.java @@ -27,12 +27,16 @@ package edu.ucsb.nceas.metacat; import java.net.ConnectException; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; +import java.util.Map; import java.util.Vector; import javax.servlet.http.HttpSession; import javax.servlet.http.HttpServletRequest; +import org.apache.commons.collections4.map.LRUMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -52,6 +56,7 @@ public class AuthSession { private HttpSession session = null; private AuthInterface authService = null; private String statusMessage = null; + private Map synchronizedGroupsCacheMap = null; private static Log logMetacat = LogFactory.getLog(AuthSession.class); /** @@ -71,6 +76,25 @@ public AuthSession() throws InstantiationException, IllegalAccessException, Clas } this.authService = (AuthInterface) createObject(authClass); } + + /** + * Constructor with cached group information for users. + * @param groupCacheSize the size of the LRUMap map + * @throws InstantiationException + * @throws IllegalAccessException + * @throws ClassNotFoundException + */ + public AuthSession(int groupCacheSize) throws InstantiationException, IllegalAccessException, ClassNotFoundException { + try { + this.authClass = PropertyService.getProperty("auth.class"); + } catch (PropertyNotFoundException e) { + logMetacat.error("AuthSession.constructor - " + e.getMessage()); + } + this.authService = (AuthInterface) createObject(authClass); + LRUMap LRUMap = new LRUMap(groupCacheSize); + synchronizedGroupsCacheMap = Collections.synchronizedMap(LRUMap); + } + /** * Get the new session @@ -287,16 +311,33 @@ private String formatOutput(String tag, String message, String sessionId, * @return null if no groups were found for the userDN */ public String[] getGroups(String logInUserName, String logInUserPassword, String userDN) throws Exception{ - String[][] groupsWithDescription = authService.getGroups(logInUserName, - logInUserPassword, userDN); - String groups[] = null; - if(groupsWithDescription != null) { - groups = new String[groupsWithDescription.length]; - for (int i = 0; i < groupsWithDescription.length; i++) { - groups[i] = groupsWithDescription[i][0]; - logMetacat.debug("AuthSession.getGroups - found that user "+userDN+" is the member of the group "+groups[i]); - } - } + String groups[] = null; + boolean lookUpLDAP = true; + if (synchronizedGroupsCacheMap != null) { + if (synchronizedGroupsCacheMap.containsKey(userDN)) { + groups = synchronizedGroupsCacheMap.get(userDN); + lookUpLDAP = false; //we got the group information, so will skip the process looking up the ldap server + logMetacat.debug("AuthSession.getGroups - get the group information for the user " + userDN + + " from the cache and it has groups - " + Arrays.toString(groups)); + } + } + if (lookUpLDAP) { + String[][] groupsWithDescription = authService.getGroups(logInUserName, + logInUserPassword, userDN); + if(groupsWithDescription != null) { + groups = new String[groupsWithDescription.length]; + for (int i = 0; i < groupsWithDescription.length; i++) { + groups[i] = groupsWithDescription[i][0]; + logMetacat.debug("AuthSession.getGroups - found that user "+userDN+" is the member of the group "+groups[i]); + } + } + if (synchronizedGroupsCacheMap != null) { + //cache is enabled, so Metacat puts the group information to the map + synchronizedGroupsCacheMap.put(userDN, groups); + logMetacat.debug("AuthSession.getGroups - put the user " + userDN + " with the group information " + + Arrays.toString(groups) + " into the cache." ); + } + } return groups; } From d59a547047554f52650e39ab5795b89f5815b21f Mon Sep 17 00:00:00 2001 From: Jing Tao Date: Mon, 30 Aug 2021 11:14:20 -0700 Subject: [PATCH 34/45] Apply the group information cache on DataONE apis --- lib/metacat.properties | 2 ++ src/edu/ucsb/nceas/metacat/AuthSession.java | 2 +- src/edu/ucsb/nceas/metacat/restservice/D1ResourceHandler.java | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/metacat.properties b/lib/metacat.properties index 02c703576..35a0a3560 100755 --- a/lib/metacat.properties +++ b/lib/metacat.properties @@ -233,6 +233,8 @@ auth.moderators= #auth.moderators=cn=knb-prod,o=NCEAS,dc=ecoinformatics,dc=org #auth.moderators=cn=knb-prod,o=NCEAS,dc=ecoinformatics,dc=org:cn=esa-moderators,dc=ecoinformatics,dc=org:cn=parc-moderators,o=PARC,dc=ecoinformatics,dc=org auth.defaultUserManagementPage=/style/common/default-user-management.jsp +# the size of the cache storing group information in the AuthSession class +auth.groupCacheSize=200 #####File-based Authentication###### auth.file.path=/var/metacat/users/password.xml auth.file.hashClassName=edu.ucsb.nceas.metacat.authentication.AuthFileBCryptHash diff --git a/src/edu/ucsb/nceas/metacat/AuthSession.java b/src/edu/ucsb/nceas/metacat/AuthSession.java index 9b9ebfd94..ea2091c57 100755 --- a/src/edu/ucsb/nceas/metacat/AuthSession.java +++ b/src/edu/ucsb/nceas/metacat/AuthSession.java @@ -334,7 +334,7 @@ public String[] getGroups(String logInUserName, String logInUserPassword, String if (synchronizedGroupsCacheMap != null) { //cache is enabled, so Metacat puts the group information to the map synchronizedGroupsCacheMap.put(userDN, groups); - logMetacat.debug("AuthSession.getGroups - put the user " + userDN + " with the group information " + + logMetacat.debug("AuthSession.getGroups - Metacat got the group information for the user " + userDN + " from LDAP and put " + Arrays.toString(groups) + " into the cache." ); } } diff --git a/src/edu/ucsb/nceas/metacat/restservice/D1ResourceHandler.java b/src/edu/ucsb/nceas/metacat/restservice/D1ResourceHandler.java index b195451ef..8f464128a 100644 --- a/src/edu/ucsb/nceas/metacat/restservice/D1ResourceHandler.java +++ b/src/edu/ucsb/nceas/metacat/restservice/D1ResourceHandler.java @@ -115,7 +115,7 @@ public class D1ResourceHandler { protected static final String FUNCTION_NAME_UPDATE = "update"; protected static AuthSession auth = null; - + protected static int authCacheSzie = Settings.getConfiguration().getInt("auth.groupCacheSize", 100); protected static boolean enableAppendLdapGroups = Settings.getConfiguration().getBoolean("dataone.session.appendLdapGroups.enabled", true); protected ServletContext servletContext; @@ -208,7 +208,7 @@ public void handle(byte httpVerb) { try { synchronized (D1ResourceHandler.class) { if (auth == null) { - auth = new AuthSession(); + auth = new AuthSession(authCacheSzie); } } groups = auth.getGroups(username, password, dn); From 97ed8595ce362834645be4d34388a92d523b5762 Mon Sep 17 00:00:00 2001 From: Jing Tao Date: Tue, 31 Aug 2021 12:10:27 -0700 Subject: [PATCH 35/45] Add the service restriction to the node capacity. --- .../nceas/metacat/dataone/MNodeService.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java b/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java index 6903eafae..8336ce0f0 100644 --- a/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java +++ b/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java @@ -129,6 +129,7 @@ import org.dataone.service.types.v1.ReplicationStatus; import org.dataone.service.types.v1.Schedule; import org.dataone.service.types.v1.Service; +import org.dataone.service.types.v1.ServiceMethodRestriction; import org.dataone.service.types.v1.Services; import org.dataone.service.types.v1.Session; import org.dataone.service.types.v1.Subject; @@ -175,6 +176,7 @@ import edu.ucsb.nceas.metacat.object.handler.NonXMLMetadataHandlers; import edu.ucsb.nceas.metacat.properties.PropertyService; import edu.ucsb.nceas.metacat.shared.MetacatUtilException; +import edu.ucsb.nceas.metacat.util.AuthUtil; import edu.ucsb.nceas.metacat.util.DocumentUtil; import edu.ucsb.nceas.metacat.util.SkinUtil; import edu.ucsb.nceas.metacat.util.SystemUtil; @@ -1302,6 +1304,7 @@ public Node getCapabilities() List mnPackageServiceAvailables = null; List mnQueryServiceAvailables = null; List mnViewServiceAvailables = null; + Vector allowedSubmitters = null; try { // get the properties of the node based on configuration information @@ -1314,6 +1317,7 @@ public Node getCapabilities() nodeType = NodeType.convert(nodeTypeString); nodeSynchronize = new Boolean(Settings.getConfiguration().getString("dataone.nodeSynchronize")).booleanValue(); nodeReplicate = new Boolean(Settings.getConfiguration().getString("dataone.nodeReplicate")).booleanValue(); + allowedSubmitters = AuthUtil.getAllowedSubmitters(); // Set the properties of the node based on configuration information and // calls to current status methods @@ -1406,6 +1410,20 @@ public Node getCapabilities() sMNStorage.setName("MNStorage"); sMNStorage.setVersion(version); sMNStorage.setAvailable(available); + if (allowedSubmitters != null && !allowedSubmitters.isEmpty()) { + ServiceMethodRestriction createRestriction = new ServiceMethodRestriction(); + createRestriction.setMethodName("create"); + ServiceMethodRestriction updateRestriction = new ServiceMethodRestriction(); + updateRestriction.setMethodName("update"); + for (int j=0; j Date: Tue, 31 Aug 2021 15:39:03 -0700 Subject: [PATCH 36/45] Add the code to test restriction service on the getCapacity method. --- .../metacat/dataone/MNodeServiceTest.java | 65 ++++++++++++++++++- 1 file changed, 62 insertions(+), 3 deletions(-) diff --git a/test/edu/ucsb/nceas/metacat/dataone/MNodeServiceTest.java b/test/edu/ucsb/nceas/metacat/dataone/MNodeServiceTest.java index a8e9ad5b3..ecbe692a8 100644 --- a/test/edu/ucsb/nceas/metacat/dataone/MNodeServiceTest.java +++ b/test/edu/ucsb/nceas/metacat/dataone/MNodeServiceTest.java @@ -107,6 +107,9 @@ import org.dataone.service.types.v1.Permission; import org.dataone.service.types.v1.Person; import org.dataone.service.types.v1.ReplicationPolicy; +import org.dataone.service.types.v1.Service; +import org.dataone.service.types.v1.ServiceMethodRestriction; +import org.dataone.service.types.v1.Services; import org.dataone.service.types.v1.Session; import org.dataone.service.types.v1.Subject; import org.dataone.service.types.v1.SubjectInfo; @@ -1281,15 +1284,68 @@ public void testListObjects() { } } - public void testGetCapabilities() { + public void testGetCapabilities() throws Exception { printTestHeader("testGetCapabilities"); + String originAllowedSubmitters = PropertyService.getInstance().getProperty("auth.allowedSubmitters"); try { Node node = MNodeService.getInstance(request).getCapabilities(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); TypeMarshaller.marshalTypeToOutputStream(node, baos); assertNotNull(node); - // TODO: should probably test other parts of the node information - + // check the service restriction. First, there is no any service restrictions + Services services = node.getServices(); + List list = services.getServiceList(); + boolean hasV1MNStorage = false; + boolean hasV2MNStorage = false; + for (Service service : list) { + if (service.getName().equals("MNStorage") && service.getVersion().equals("v1")) { + hasV1MNStorage = true; + List restrictions = service.getRestrictionList(); + assertTrue(restrictions == null || restrictions.isEmpty()); + } + if (service.getName().equals("MNStorage") && service.getVersion().equals("v2")) { + hasV2MNStorage = true; + List restrictions = service.getRestrictionList(); + assertTrue(restrictions == null || restrictions.isEmpty()); + } + } + assertTrue(hasV1MNStorage); + assertTrue(hasV2MNStorage); + // check the service restriction. Second, there are some service restrctions + PropertyService.getInstance().setPropertyNoPersist("auth.allowedSubmitters", + "http\\://orcid.org/0000-0002-1209-5268:cn=parc,o=PARC,dc=ecoinformatics,dc=org"); + AuthUtil.populateAllowedSubmitters();//make the allowedSubimtters effective + node = MNodeService.getInstance(request).getCapabilities(); + services = node.getServices(); + list = services.getServiceList(); + hasV1MNStorage = false; + hasV2MNStorage = false; + for (Service service : list) { + if (service.getName().equals("MNStorage") && service.getVersion().equals("v1")) { + hasV1MNStorage = true; + ServiceMethodRestriction restriction1 = service.getRestriction(0); + assertTrue(restriction1.getMethodName().equals("create")); + assertTrue(restriction1.getSubject(0).getValue().equals("http://orcid.org/0000-0002-1209-5268")); + assertTrue(restriction1.getSubject(1).getValue().equals("cn=parc,o=PARC,dc=ecoinformatics,dc=org")); + ServiceMethodRestriction restriction2 = service.getRestriction(1); + assertTrue(restriction2.getMethodName().equals("update")); + assertTrue(restriction2.getSubject(0).getValue().equals("http://orcid.org/0000-0002-1209-5268")); + assertTrue(restriction2.getSubject(1).getValue().equals("cn=parc,o=PARC,dc=ecoinformatics,dc=org")); + } + if (service.getName().equals("MNStorage") && service.getVersion().equals("v2")) { + hasV2MNStorage = true; + ServiceMethodRestriction restriction1 = service.getRestriction(0); + assertTrue(restriction1.getMethodName().equals("create")); + assertTrue(restriction1.getSubject(0).getValue().equals("http://orcid.org/0000-0002-1209-5268")); + assertTrue(restriction1.getSubject(1).getValue().equals("cn=parc,o=PARC,dc=ecoinformatics,dc=org")); + ServiceMethodRestriction restriction2 = service.getRestriction(1); + assertTrue(restriction2.getMethodName().equals("update")); + assertTrue(restriction2.getSubject(0).getValue().equals("http://orcid.org/0000-0002-1209-5268")); + assertTrue(restriction2.getSubject(1).getValue().equals("cn=parc,o=PARC,dc=ecoinformatics,dc=org")); + } + } + assertTrue(hasV1MNStorage); + assertTrue(hasV2MNStorage); } catch (MarshallingException e) { e.printStackTrace(); fail("The node instance couldn't be parsed correctly:" + e.getMessage()); @@ -1302,6 +1358,9 @@ public void testGetCapabilities() { e.printStackTrace(); fail("Probably not yet implemented: " + e.getMessage()); + } finally { + PropertyService.getInstance().setPropertyNoPersist("auth.allowedSubmitters", originAllowedSubmitters); + AuthUtil.populateAllowedSubmitters();//make the allowedSubimtters effective } } From 49dc33c4a1677e4d7f4c970f4663ac2de3e28056 Mon Sep 17 00:00:00 2001 From: Jing Tao Date: Thu, 2 Sep 2021 15:30:34 -0700 Subject: [PATCH 37/45] Merge the schema.org change peter did into metacat. --- metacat-index/pom.xml | 2 +- .../resources/application-context-json-ld.xml | 10 ++ .../application-context-schema-org.xml | 109 +++++++++++++++++- 3 files changed, 118 insertions(+), 3 deletions(-) diff --git a/metacat-index/pom.xml b/metacat-index/pom.xml index d86ef299d..8206535e6 100644 --- a/metacat-index/pom.xml +++ b/metacat-index/pom.xml @@ -9,7 +9,7 @@ http://maven.apache.org - 2.3.13 + 2.3.14 2.15.0 diff --git a/metacat-index/src/main/resources/application-context-json-ld.xml b/metacat-index/src/main/resources/application-context-json-ld.xml index 59507c323..252d5028c 100644 --- a/metacat-index/src/main/resources/application-context-json-ld.xml +++ b/metacat-index/src/main/resources/application-context-json-ld.xml @@ -28,6 +28,15 @@ + + + + + + + + + @@ -42,6 +51,7 @@ + diff --git a/metacat-index/src/main/resources/application-context-schema-org.xml b/metacat-index/src/main/resources/application-context-schema-org.xml index ddb30ee90..a34ba769e 100644 --- a/metacat-index/src/main/resources/application-context-schema-org.xml +++ b/metacat-index/src/main/resources/application-context-schema-org.xml @@ -1,5 +1,6 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + PREFIX xsd: + SELECT DISTINCT (str(?string) as ?text) + WHERE { + { + ?a ?b ?string . + filter(datatype(?string) = xsd:string) . + } + UNION + { + ?a ?b ?string . + filter(datatype(?string) = SO:HTML) . + } + } + ]]> + + + From 20ec03b16c5e502458a06fd70e140b5430231d87 Mon Sep 17 00:00:00 2001 From: Bryce Mecum Date: Thu, 2 Sep 2021 19:11:10 -0800 Subject: [PATCH 38/45] Update OBOE from 1.0 to 1.2 Closes # --- ...ication-context-ontology-model-service.xml | 48 +- .../resources/ontologies/oboe-anatomy.owl | 0 .../resources/ontologies/oboe-biology.owl | 0 .../ontologies/oboe-characteristics.owl | 443 +++--- .../resources/ontologies/oboe-chemistry.owl | 0 .../main/resources/ontologies/oboe-core.owl | 1275 ++++++++--------- .../resources/ontologies/oboe-ecology.owl | 0 .../resources/ontologies/oboe-environment.owl | 0 .../resources/ontologies/oboe-spatial.owl | 523 ++++--- .../resources/ontologies/oboe-standards.owl | 1267 ++++++++-------- .../main/resources/ontologies/oboe-taxa.owl | 0 .../resources/ontologies/oboe-temporal.owl | 116 +- .../src/main/resources/ontologies/oboe.owl | 53 +- 13 files changed, 1800 insertions(+), 1925 deletions(-) mode change 100644 => 100755 metacat-index/src/main/resources/ontologies/oboe-anatomy.owl mode change 100644 => 100755 metacat-index/src/main/resources/ontologies/oboe-biology.owl mode change 100644 => 100755 metacat-index/src/main/resources/ontologies/oboe-characteristics.owl mode change 100644 => 100755 metacat-index/src/main/resources/ontologies/oboe-chemistry.owl mode change 100644 => 100755 metacat-index/src/main/resources/ontologies/oboe-core.owl mode change 100644 => 100755 metacat-index/src/main/resources/ontologies/oboe-ecology.owl mode change 100644 => 100755 metacat-index/src/main/resources/ontologies/oboe-environment.owl mode change 100644 => 100755 metacat-index/src/main/resources/ontologies/oboe-spatial.owl mode change 100644 => 100755 metacat-index/src/main/resources/ontologies/oboe-standards.owl mode change 100644 => 100755 metacat-index/src/main/resources/ontologies/oboe-taxa.owl mode change 100644 => 100755 metacat-index/src/main/resources/ontologies/oboe-temporal.owl mode change 100644 => 100755 metacat-index/src/main/resources/ontologies/oboe.owl diff --git a/metacat-index/src/main/resources/application-context-ontology-model-service.xml b/metacat-index/src/main/resources/application-context-ontology-model-service.xml index cb11550f1..f1f00d750 100644 --- a/metacat-index/src/main/resources/application-context-ontology-model-service.xml +++ b/metacat-index/src/main/resources/application-context-ontology-model-service.xml @@ -16,18 +16,18 @@ http://purl.dataone.org/ontologies/observation/d1-ECSO.owl http://purl.dataone.org/ontologies/provenance/ProvONE/v1/owl/provone.owl http://purl.obolibrary.org/obo/envo.owl - http://ecoinformatics.org/oboe/oboe.1.0/oboe-owl.owl - http://ecoinformatics.org/oboe/oboe.1.0/oboe-core.owl - http://ecoinformatics.org/oboe/oboe.1.0/oboe-characteristics.owl - http://ecoinformatics.org/oboe/oboe.1.0/oboe-standards.owl - http://ecoinformatics.org/oboe/oboe.1.0/oboe-spatial.owl - http://ecoinformatics.org/oboe/oboe.1.0/oboe-temporal.owl - http://ecoinformatics.org/oboe/oboe.1.0/oboe-taxa.owl - http://ecoinformatics.org/oboe/oboe.1.0/oboe-biology.owl - http://ecoinformatics.org/oboe/oboe.1.0/oboe-ecology.owl - http://ecoinformatics.org/oboe/oboe.1.0/oboe-environment.owl - http://ecoinformatics.org/oboe/oboe.1.0/oboe-chemistry.owl - http://ecoinformatics.org/oboe/oboe.1.0/oboe-anatomy.owl + http://ecoinformatics.org/oboe/oboe.1.2/oboe-owl.owl + http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl + http://ecoinformatics.org/oboe/oboe.1.2/oboe-characteristics.owl + http://ecoinformatics.org/oboe/oboe.1.2/oboe-standards.owl + http://ecoinformatics.org/oboe/oboe.1.2/oboe-spatial.owl + http://ecoinformatics.org/oboe/oboe.1.2/oboe-temporal.owl + http://ecoinformatics.org/oboe/oboe.1.2/oboe-taxa.owl + http://ecoinformatics.org/oboe/oboe.1.2/oboe-biology.owl + http://ecoinformatics.org/oboe/oboe.1.2/oboe-ecology.owl + http://ecoinformatics.org/oboe/oboe.1.2/oboe-environment.owl + http://ecoinformatics.org/oboe/oboe.1.2/oboe-chemistry.owl + http://ecoinformatics.org/oboe/oboe.1.2/oboe-anatomy.owl http://ecoinformatics.org/oboe-ext/sbclter.1.0/oboe-sbclter.owl @@ -41,18 +41,18 @@ - - - - - - - - - - - - + + + + + + + + + + + + diff --git a/metacat-index/src/main/resources/ontologies/oboe-anatomy.owl b/metacat-index/src/main/resources/ontologies/oboe-anatomy.owl old mode 100644 new mode 100755 diff --git a/metacat-index/src/main/resources/ontologies/oboe-biology.owl b/metacat-index/src/main/resources/ontologies/oboe-biology.owl old mode 100644 new mode 100755 diff --git a/metacat-index/src/main/resources/ontologies/oboe-characteristics.owl b/metacat-index/src/main/resources/ontologies/oboe-characteristics.owl old mode 100644 new mode 100755 index 943d5c816..ad139966f --- a/metacat-index/src/main/resources/ontologies/oboe-characteristics.owl +++ b/metacat-index/src/main/resources/ontologies/oboe-characteristics.owl @@ -9,26 +9,28 @@ - + + + ]> - - + OBOE Characteristics - This ontology contains general terms for characteristics that are measured either quantitatively or qualitatively, including physical characteristics, behavioral characteristics, and administrative characteristics. - Copyright (c) 2006-2011 The Regents of the University of California. All rights reserved. This work is licensed under the Creative Commons Attribution 3.0 Unported License. To view a copy of this license, visit http://creativecommons.org/licenses/by/3.0/ or send a letter to Creative Commons, 444 Castro Street, Suite 900, Mountain View, California, 94041, USA. - Version 1.0 - + This ontology contains general terms for (physical) characteristics that are measured either quantitatively or qualitatively. + Copyright (c) 2006-2016 The Regents of the University of California. All rights reserved. This work is licensed under the Creative Commons Attribution 3.0 Unported License. To view a copy of this license, visit http://creativecommons.org/licenses/by/3.0/ or send a letter to Creative Commons, 444 Castro Street, Suite 900, Mountain View, California, 94041, USA. + Version 1.2 + @@ -66,560 +68,547 @@ - + - - Algae Life Stage - - The developmental stage of a given algae. + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - - - - - - - - - + + measures how easily electricity flows along a certain path through an electrical element. The SI derived unit for electrical conductance is the siemens - + - - + + is a measure of a material's ability to conduct an electric current. Units are siemens per meter - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - Fish Life Stage - - The developmental stage of a given fish. + + - + - - + + - + - - + + + provides an identifying sequence of characters to refer to an entity within a context. An Identifier can be unique within a local context or a global context. - + - - + + - + - - + + - + - - + + Irradiance, radiant emittance, and radiant exitance are radiometry terms for the power per unit area of electromagnetic radiation at a surface. "Irradiance" is used when the electromagnetic radiation is incident on the surface. "Radiant exitance" or "radiant emittance" is used when the radiation is emerging from the surface. The SI units for all of these quantities are watts per square meter (W/m2) (wikipedia). All of these quantities characterize the total amount of radiation present, at all frequencies. It is also common to consider each frequency in the spectrum separately. When this is done for radiation incident on a surface, it is called spectral irradiance, and has SI units W/m3, or commonly W·m−2·nm−1. - + - - + + - + - + Life Stage - + The developmental stage of life of a given organism. - + - - + + Luminance is a photometric measure of the luminous intensity per unit area of light travelling in a given direction. It describes the amount of light that passes through or is emitted from a particular area, and falls within a given solid angle. The SI unit for luminance is candela per square metre (cd/m2). - + - - + + - + - - + + - + - - + + - + - - + + - + - - Mammal Life Stage - - The developmental stage of a given mammal. + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + + Molality amount of substance of solute B in a solution divided by the mass of the solvent: Often favored because measurements of mass are more precise than measurements of volume. - + - - - Molality amount of substance of solute B in a solution divided by the mass of the solvent: Often favored because measurements of mass are more precise than measurements of volume. + + + provides a character phrase to refer to an entity. A Name is typically used to reference an entity. For example, the Name for a particular instance of a lake might be 'Lake Tahoe'. - + - - + + The number of photons striking an area per unit time. This could be probably also be called a PhotonArealDensityRate. - + - - Plant Life Stage - - The developmental stage of a given plant. + + - + - - + + - + - - + + - + - - + + + Radiance and spectral radiance are radiometric measures that describe the amount of light that passes through or is emitted from a particular area, and falls within a given solid angle in a specified direction. They are used to characterize both emission from diffuse sources and reflection from diffuse surfaces. The SI unit of radiance is watts per steradian per square metre (W·sr−1·m−2). - + - - - Radiance and spectral radiance are radiometric measures that describe the amount of light that passes through or is emitted from a particular area, and falls within a given solid angle in a specified direction. They are used to characterize both emission from diffuse sources and reflection from diffuse surfaces. The SI unit of radiance is watts per steradian per square metre (W·sr−1·m−2). + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + + Irradiance, radiant emittance, and radiant exitance are radiometry terms for the power per unit area of electromagnetic radiation at a surface. "Irradiance" is used when the electromagnetic radiation is incident on the surface. "Radiant exitance" or "radiant emittance" is used when the radiation is emerging from the surface. The SI units for all of these quantities are watts per square meter (W/m2) (wikipedia). All of these quantities characterize the total amount of radiation present, at all frequencies. It is also common to consider each frequency in the spectrum separately. When this is done for radiation incident on a surface, it is called spectral irradiance, and has SI units W/m3, or commonly W·m−2·nm−1. - + - - - Irradiance, radiant emittance, and radiant exitance are radiometry terms for the power per unit area of electromagnetic radiation at a surface. "Irradiance" is used when the electromagnetic radiation is incident on the surface. "Radiant exitance" or "radiant emittance" is used when the radiation is emerging from the surface. The SI units for all of these quantities are watts per square meter (W/m2) (wikipedia). All of these quantities characterize the total amount of radiation present, at all frequencies. It is also common to consider each frequency in the spectrum separately. When this is done for radiation incident on a surface, it is called spectral irradiance, and has SI units W/m3, or commonly W·m−2·nm−1. + + - + - - + + - + - - + + + provides an alphanumeric string of characters to refer to an entity. A TagNumber is typically used to uniquely identify an entity such as a sample or an individual specimen such as a tree or a rock sample. - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + - + diff --git a/metacat-index/src/main/resources/ontologies/oboe-chemistry.owl b/metacat-index/src/main/resources/ontologies/oboe-chemistry.owl old mode 100644 new mode 100755 diff --git a/metacat-index/src/main/resources/ontologies/oboe-core.owl b/metacat-index/src/main/resources/ontologies/oboe-core.owl old mode 100644 new mode 100755 index 7660b4867..b63f2f24e --- a/metacat-index/src/main/resources/ontologies/oboe-core.owl +++ b/metacat-index/src/main/resources/ontologies/oboe-core.owl @@ -1,35 +1,22 @@ - - - - - - - - - - -]> - - - - + xmlns:swrl="http://www.w3.org/2003/11/swrl#" + xmlns:xsd="http://www.w3.org/2001/XMLSchema#" + xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#"> + + http://creativecommons.org/licenses/by/3.0/ oboe-core - '$Id: Observation.owl,v 1.29 2008/01/30 04:36:12 bowers Exp $' - http://creativecommons.org/licenses/by/3.0/ - Version 1.0 pre-release - Copyright (c) 2006-2011 The Regents of the University of California. All rights reserved. This work is licensed under the Creative Commons Attribution 3.0 Unported License. To view a copy of this license, visit http://creativecommons.org/licenses/by/3.0/ or send a letter to Creative Commons, 444 Castro Street, Suite 900, Mountain View, California, 94041, USA. This ontology provides the core OBOE modeling constructs. + Version 1.2 + Copyright (c) 2006-2016 The Regents of the University of California. All rights reserved. This work is licensed under the Creative Commons Attribution 3.0 Unported License. To view a copy of this license, visit http://creativecommons.org/licenses/by/3.0/ or send a letter to Creative Commons, 444 Castro Street, Suite 900, Mountain View, California, 94041, USA. @@ -42,17 +29,37 @@ /////////////////////////////////////////////////////////////////////////////////////// --> - - - A class annotation property that states that the annotated class is conceptually similar (e.g., overlapping), but not identical to the concept identified by the value of the annotation property. If the value of the annotation property denotes an OWL class, then the similarConceptAsClass annotation property should be used instead. - - + + + + + + A class annotation property that states that the annotated class is conceptually the same class as (i.e., conceptually equivalent to) the concept identified in the annotation value. If the value of the annotation property denotes an OWL class, then the sameConceptAsClass annotation property should be used instead. - + + + + + + A class annotation property that states that the annotated class is conceptually the same class as (i.e., conceptually equivalent to) the referenced class (as given by the value of the annotation property). Note that being conceptually the same can differ from strict class equivalance in that the two classes may have different structural definitions within their specific ontologies, e.g., their superclasses and subclasses in the respective ontologies may differ. - + + + + + + + + A class annotation property that states that the annotated class is conceptually similar (e.g., overlapping), but not identical to the concept identified by the value of the annotation property. If the value of the annotation property denotes an OWL class, then the similarConceptAsClass annotation property should be used instead. + + + + + + + A class annotation property that states that the annotated class is conceptually similar (e.g., overlapping), but not identical to the referenced class (the value of the annotation property). @@ -71,7 +78,7 @@ - + @@ -86,318 +93,269 @@ - + - - - The contextFor property states that one observation serves as the context for another observation. In the case of contextFor, the domain is the context, and the range is the observation being contextualized. Context defines a semantic relationship between two entities that is a fundamental aspect of the observations, but not necessarily of the entities themselves. For example, most measurements are accomplished in a spatio-temporal framework that might be valuable context. The assertions made by contextual observations are assumed of the contextualized observations. Context is a transitive relationship. - - - - - - - + + + + + The characteristicFor property gives the entity of the characteristic. - + - - - The contextObservationFor property relates an contextualizing observation to the corresponding context measurement. - - + + + + The entityRelated property asserts that the entity of one observation has an observed relationship to the entity in another observation. - + - - + + + + The hasBaseQualifier together with the hasNextQualfiier property allows qualifier characteristics to be combined and ordered. For example, for an MinimumDailyAverage qualifier, the hasBaseQualifier refers to Minimum and the hasNextQualifier refers to a DailyAverage qualifier, and for a DailyAverage qualifier, the hasBaseQualifier refers to a Daily qualifier and the hasNextQualifier refers to an Average qualifier. - - - - - - - - - - - The hasContext property states that one observation serves as the context for another observation. In the case of hasContext, the domain is the observation being contextualized, and the range is the context. Context defines a semantic relationship between two entities that is a fundamental aspect of the observations, but not necessarily of the entities themselves. For example, most measurements are accomplished in a spatio-temporal framework that might be valuable context. The assertions made by contextual observations are assumed of the contextualized observations. Context is a transitive relationship. - - - - - - - + - - - The hasContextObservation gives the contextualizing observation for the context. - - + + + + + The hasContext property asserts that one observation serves as the context for another observation. In a hasContext property the domain is the observation being contextualized and the range is the context. Context defines a semantic relationship between two entities that is a fundamental aspect of the observations, but not necessarily of the entities themselves. For example, most measurements are accomplished in a spatio-temporal framework that might be valuable context. The assertions made by contextual observations are assumed of the contextualized observations. Context is a transitive relationship. - + - - - The hasMeasurement property gives the measurements of the observed entity. - - + + + + + + The hasMeasurement property gives the measurements of the observed entity. - + - + + + The hasMember property gives the observations of an observation collection. - - - + - - + + + + The hasNexQualifier together with the hasBaseQualfiier property allows qualifier characteristics to be combined and ordered. For example, for an MinimumDailyAverage qualifier, the hasBaseQualifier refers to Minimum and the hasNextQualifier refers to a DailyAverage qualifier, and for a DailyAverage qualifier, the hasBaseQualifier refers to a Daily qualifier and the hasNextQualifier refers to an Average qualifier. - - - + - - - The hasObservationContext is a special type of hasMeasurement property that links an observation to a context measurement. - - - + + + + The hasObservedRelation property gives the observed relation of an observation. The observation entity is the source (head) entity of the corresponding relationship that was observed. - + - - - The hasQualifier property assigns an optional characteristic qualifier to a physical characteristic. Examples of qualifiers are Average, Minimum, Maximum, Daily, etc. - - + + + + + The hasQualifier property assigns an optional characteristic qualifier to a characteristic. Examples of qualifiers are Average, Minimum, Maximum, Daily, etc. - + - - + + + + The hasSourceUnit property gives the source unit being converted from within a unit conversion. - - - + - - + + + + The hasTargetUnit property gives the target unit being converted to within a unit conversion. - - - + - - The hasUnit property gives the underlying base or derived unit for a derived or composite unit, respectively. - + + - - + + + - - + + - - + + + - + + The hasUnit property gives the underlying base or derived unit for a derived or composite unit, respectively. - + - - + + + + The hasValue property gives the value of the measurement. - - - + - - - The measuredBy property gives the measurement a characteristic is used in. - - - + + + + + The measurementFor property gives the observation (observed entitiy) that a measurement is associated with. Each measurement is for exactly one observation. - + - - - The measurementFor property relates a measurement to its corresponding observation. - - - + + + + The containsMeasurementsOfType property gives the MeasurementType to which measured values in some container belong. The domain is unrestricted to allow the property to be used to annotate any containers holding measured values, but its range is restricted to only subclasses of MeasurementType. If, for example, an attribute containsMeasurementsOfType "AirTemperatureType", then one can infer that the values in that attribute all represent measures of the associated Characteristic (e.g., Temperature) of the associated Entity (e.g., Air), or whatever constraints are imposed in the definition of the MeasurementType. - + - - The memberOf property gives the observation collections the observation is a member of. - - - + + + + + The measuresCharacteristic property gives the Characteristic that would be measured by a MeasurementType. - + - - - The observationContextFor property gives the contextualized observation for a context measurement. - - - - + + + + + The measuresEntity property gives the Entity that would be measured by a MeasurementType. - + - - - The observedBy property gives the observation of an observed entity. - - - + + + + + The measuresUsingProtocol property gives the Protocol that would be used by a MeasurementType. - + - - - The ofCharacteristic property gives the entity characteristic measured by a measurement. - - + + + + + The measuresUsingStandard property gives the Standard that would be used by a MeasurementType. - - - - - The ofEntity property gives the observed entity of an observation. - - - - + - - - - - - The protocolFor property gives the measurement that uses a protocol. - - - + + + + + The ofCharacteristic property gives the entity characteristic measured by a measurement. - + - - - The standardFor property gives the measurements that use this standard. - - - + + + + + The ofEntity property gives the observed entity of an observation. - + - - + + + + + The usesProtocol property gives the protocol used in a measurement. - - - + - - - The usesStandard property gives the standard (e.g., unit) used in a measurement. - - - - - - - - - - - The valueFor property gives the measurement a value is used in. - - - + + + + + + The usesStandard property gives the standard (e.g., unit) used in a measurement. @@ -413,68 +371,76 @@ - + - - + + + The hasCode property gives the underlying value (or coded representation) of a primitive value. - - - + - - + + + + The hasMultiplier property gives the multiplier value in a unit conversion. - - - + - - + + + + The hasOffset property gives the offset value in a unit conversion. - - - + - - + + + + The hasPower property gives the power a base unit is raised to within a derived unit. - - - + - - - This property gives the precision of a measurement value. - - + + + + + This property gives the precision of a measurement value. - + - - + + + + + + + + + + + + + The usesMethod property gives the actual method used to carry out the measurement. - - @@ -490,200 +456,150 @@ - + - + + + A base qualifier denotes a simple, atomic qualifier such as average, minimum, and maximum. Base Characteristic Qualifier - - A Base Qualifier denotes a simple, atomic qualifier such as average, minimum, maximum, etc. - + - + + + + + + A base unit is a unit that is not naturally decomposed into other units. Base Unit - - A BaseUnit simply represents a unit that is not naturally decomposed into other units. - + - - Boolean - - - - Represents a Boolean value, either 'true' or 'false'. + + + + - + - + + + + + + + + + + + + + + + + + A characteristic represents a property of an entity that can be measured (e.g., height, length, or color). A characteristic of an entity is observed through a measurement, which further asserts a value of the characteristic for the entity. A characteristic type (e.g., "height") can be associated with many different entities, whereas an individual characteristic (a particular occurrence of the "height" characteristic) is associated to exactly one entity. Characteristic - - - - - - - - - - - - - - - - - - A characteristic represents a property of an entity that can be measured (e.g., height, length, or color). We adopt the standard distinction (e.g., as in M. Bunge, 1979) between things (entities) and their properties (characteristics). A characteristic of an entity is observed through a measurement, which further asserts a value of the characteristic for the entity. A characteristic type (e.g., "height") can be associated with many different entities, whereas an individual characteristic (a particular occurrence of the "height" characteristic) is associated to exactly one entity. - + - - Characteristic Qualifier + - - + + - - - - - - - - A Characteristic Qualifier is a generic derived (e.g., computed) property of an entity that must be combined with concrete characteristics to be used in a measurement. A qualifier is either a base (atomic) or composite qualifier. As an example, the base qualifier Minimum can be combined with a characteristic Length to create a Minimum Length qualified characteristic. A composite qualifer is used to combine multiple qualifiers. + + + + + + + + + + A characteristic qualifier is a derived property of an entity that must be combined with concrete characteristics to be used in a measurement. A qualifier is either a base (atomic) or composite qualifier. As an example, the base qualifier minimum can be combined with a characteristic length to create a minimum length qualified characteristic. A composite qualifer is used to combine multiple qualifiers. + Characteristic Qualifier - + - - Composite Characteristic Qualifier - + + - - - 1 + + 1 + - - - 1 + + 1 + - A Composite Qualifier is used to combine multiple qualifiers. Each composite qualifier has a base qualifier and a reference to another qualifier. Composite qualifiers are ordered (through hasNextQualifier) to distinguish between, e.g., AverageMinimum versus MinimumAverage. + A composite qualifier combines multiple qualifiers. Each composite qualifier has a base qualifier and a reference to another qualifier. Composite qualifiers are ordered (through hasNextQualifier) to distinguish between the scope of qualifiers, e.g., AverageMinimum versus MinimumAverage. + Composite Characteristic Qualifier - + - - Composite Unit - + + - + + 2 - - + + + - 2 - A CompositeUnit is a product at least two base or derived units. For example, 'meter per second square' denotes a composite unit defined over 'meter' (a base unit) and 'per second square' (a derived unit). - - - - - - - - Context - - - - - - - - - - - - - - A context measurement is a special measurement for capturing the details of a contextual relationship among two observations. These details include a characteristic that asserts the relationship between the two entities being observed. Optionally, a protocol (e.g., defining the procedure for observing the relationship) and a standard can also be given (however, these will typically not be specified as part of context). The value of the context measurement is the contextualizing entity, and the observation for the context measurement is the contextualized observation. - - - - - - - - Decimal - - - Represents any real number value (see the XSD decimal type). + + + A composite unit is the product of at least two base or derived units. For example, 'meter per second square' denotes a composite unit defined over 'meter' (a base unit) and 'per second square' (a derived unit). + Composite Unit - + - - Derived Unit - - - - - - - - - - - - - 1 - - - - - - - - + + - - 0 + + 0 @@ -692,285 +608,325 @@ - - 1 + + 1 - A DerivedUnit raises a base unit to an integer power other than 0 or 1. For example, the unit 'square meter' is a unit derived from the 'meter' base unit raised to the power 2. + + + + + + + + A derived unit raises a base or prefixed unit to an integer power other than 0 or 1. For example, the unit 'square meter' is a unit derived from the 'meter' base unit raised to the power 2. As another example, a 'square centimeter' is a unit derived from the 'centimeter' prefixed unit raised to the power 2. + +BRL 20160601: I removed this restriction so that CompositeUnits like XXXPerLiter would not reason to owl:Nothing. hasUnit exactly 1 (BaseUnit or PrefixedUnit) + Derived Unit - + - + + + + + + + + An entity is an object (e.g., a tree, a community, an ecological process). Entities constitute the foci of observations, i.e., every observation is of exactly one entity. Entity - - - - - - An entity denotes a concrete or conceptual object that has been observed (e.g., a tree, a community, an ecological process). We adopt the standard distinction (e.g., as in M. Bunge, 1979) between things (entities) and their properties (characteristics). Entities constitute the foci of observations, i.e., every observation is of exactly one entity. - + - - Measurement + + + + An identifying characteristic is used to identify or name an entity either globally or within a context, such as a name of a lake or a tag number assigned to a tree. Unlike a MeasuredCharacteristic, an IdentifyingCharacteristic is assigned and not measured. + Identifying Characteristic + + + + + + + + + + + + + + + + + + A measured characteristic of an entity is one that is measurable within the physical world. A measured characteristic includes primary and derived physical dimensions (e.g., length, mass, area, density). The measured value assigned to a measured characteristic may be a quantity (a numerical value) or a quality (a category), including both nominal and ordinal categories. + Measured Characteristic + + + + + + + + + + + + + + + A measurement value is a quantitative or qualitative result of a measurement. Measurement values can contain a coded representation of the result, e.g., as a number or string. + Measurement Value + + + + + + + - - + + - - + + - - + + - - + + - - + + 1 + - - - - - A measurement is an assertion that a characteristic of an entity was measured and/or recorded. A measurement is also composed of a value, a measurement standard, and a precision (associated with the measured value). Measurements also encapsulate characteristics that were recorded, but not necessarily measured in a physical sense. For example, the name of a location and a taxon can be captured through measurements. + + + + + + A measurement is an assertion that a characteristic of an entity had a particular value with respect to an observation event. A measurement is comprised of a characteristic, a value, a measurement standard, and a protocol. Measurements can also have precision as well as a description of the methods used. Measurements can encapsulate characteristics that were recorded, but not necessarily measured in a physical sense. For example, the name of a location and a taxon can be captured through measurements. + Measurement - + - - Name - - - - - A Name characteristic provides the name (or label) used for identifying (either globally or within a context) an entity. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + A MeasurementType describes the type of a Measurement in which the Measurement would follow the associated Protocol to record the value of the associated Characteristic of the associated Entity using the associated Standard. Any of these associated properties may be omitted, in which case the MeasurementType is only constrained by the provided associations. A MeasurementType is a hypothetical construct, in that it is not associated with a particular instance of a Measurement. + Measurement Type - + - - Observation + - - + + - - - - An observation represents an "observed entity," that is, an entity that was observed by an observer. An observation often consists of measurements that refer to one or more measured characteristics of the observed entity. Observations may also be made within a broader context. The context of an observation is given by other observations, implying that an observed entity (and specifically each corresponding measurement) contextualizes another observed entity (and its corresponding measurements). For example, an observation of a location may serve as context for an observation of an organism. In this case, the observed characteristic values of the location (such as humidity) are assumed constant for the observation of the organism. + + + + An observation is an assertion that an entity (e.g., biological organisms, geographic locations, or environmental features, among others) was observed by an observer. An observation primarily serves to group a set of measurements together into a single "observation event". Observations are often made within a broader context. The context of an observation is given by other observations, implying that an observed entity (and specifically each corresponding measurement) contextualizes another observed entity (and its corresponding measurements). For example, an observation associated with a location may serve as context for an observation associated with an organism. In this case, the observed characteristic values of the location (such as humidity) are assumed constant for the corresponding measurements of the organism. + Observation - + - + + + + An observation collection is a container for a set of observations. Observation Collection - - - An observation collection contains a set of observations. - + - - Physical Characteristic - - - - A Physical Characteristic of an Entity is one that is measurable within the physical world, and subsumes primary and derived physical dimensions (e.g., length, mass, area, density). The measured value assigned to a Physical characteristic may be a quantity (a numerical value) or a quality (a label). - - - - - - - - Primitive Value - - A PrimitiveValue is an entity that represents a literal value (e.g., a number, a string, a a Boolean, etc.) + + + + + + + + 0 + + + + + + + + 1 + + + + + + + + + + A prefixed unit consists of a base unit and a prefix integer value that serves as a multiplier. For example, a centimeter is a prefixed unit that combines a meter base unit and the prefix multiplier 0.01 such that one centimeter is equivalent to 0.01 meters. + - + - + + + + + + + + + + + + + A protocol is a procedure for generating or processing data. Protocol - - A protocol represents a specific procedure that is used for generating or processing data. - - - - Relationship - - - A Relationship represents a directional (binary) association between two entities in which the observed entity is associated to the given entity. Examples include spatial (e.g., within, overlaps, adjacent), temporal (e.g., during, before, after), and composition (e.g., part of, has part) relationships. - - + - - - - - Standard + - - + + + + + + - A standard defines a reference for comparing or naming entities via a measurement. A standard can be defined intentionally (e.g., as in the case of units) or extensionally (by listing the values of the standard, e.g., for color this might be red, blue, yellow, etc). - - - - - - - - String - - Represents any character string value. - - - - - - - - Type - - A Type characteristic provides the type (or class) of an entity as a measured (or observed) value. A typical example is a biological taxonomy designation. + A standard defines a reference for comparing or naming entities via a measurement. A standard can be defined intentionally (e.g., as in the case of units) or extensionally (by listing the values of the standard, e.g., for color this might be red, blue, yellow, etc). + Standard - + - - Unit + - - - + + + + - + + A unit is a standard quantification for physical measurements. A unit is either a base unit, a composite unit, or a derived unit. + Unit + + + + + + + - - - - - - - - - - - - + + - - - - - - - - - - - - + + - The base class for physical measurement units. - - - - - - - - Unit Conversion - - + + - - + + - A UnitConversion defines a mapping from a source unit to a target unit via a multiplier and an offset. + A unit conversion is a mapping from a source unit to a target unit via a multiplier and an offset value. + Unit Conversion - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + - + + - - + + + - + + - + + + + + + + + + + - + + - - - + + + - + + - - - - - - - - - - - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - - + + - - - - - + + + + + + - + + - - + + + - + + + + + + + + + + - - + + - - - - - + + + + + + - + + + + + + + + + + + + + + + - - - - - - - - + - - + + - - - - - + + + + + + - + + - + - + diff --git a/metacat-index/src/main/resources/ontologies/oboe-ecology.owl b/metacat-index/src/main/resources/ontologies/oboe-ecology.owl old mode 100644 new mode 100755 diff --git a/metacat-index/src/main/resources/ontologies/oboe-environment.owl b/metacat-index/src/main/resources/ontologies/oboe-environment.owl old mode 100644 new mode 100755 diff --git a/metacat-index/src/main/resources/ontologies/oboe-spatial.owl b/metacat-index/src/main/resources/ontologies/oboe-spatial.owl old mode 100644 new mode 100755 index 0b32a2057..71c12f9ed --- a/metacat-index/src/main/resources/ontologies/oboe-spatial.owl +++ b/metacat-index/src/main/resources/ontologies/oboe-spatial.owl @@ -9,26 +9,28 @@ - + + + ]> - - + OBOE Spatial This ontology contains terms relating to spatial concepts that are common across OBOE extensions. The terms are derived from the International Standards Organization (ISO) geospatial standards in the 19100 series and from the Geography Markup Language version 3.2.1. - Copyright (c) 2006-2011 The Regents of the University of California. All rights reserved. This work is licensed under the Creative Commons Attribution 3.0 Unported License. To view a copy of this license, visit http://creativecommons.org/licenses/by/3.0/ or send a letter to Creative Commons, 444 Castro Street, Suite 900, Mountain View, California, 94041, USA. - Version 1.0 - + Copyright (c) 2006-2012 The Regents of the University of California. All rights reserved. This work is licensed under the Creative Commons Attribution 3.0 Unported License. To view a copy of this license, visit http://creativecommons.org/licenses/by/3.0/ or send a letter to Creative Commons, 444 Castro Street, Suite 900, Mountain View, California, 94041, USA. + Version 1.1 + @@ -66,500 +68,481 @@ - + - - Plot - - - - - - - - - Replicate Plot - - - - - - - - - ReplicateTransect - - - - - - - - - Transect - - - - - - - - - Waypoint - - waypoint is usually defined by a latitude-longitude pair. It often defines the nominal location of a station or sampling place. - - - - - - - + Affine Coordinate System - + - + - + Cartesian Coordinate System - + - + - + Composite Curve - + - + - + Composite Solid - + - + - + Composite Surface - + - + - + Compound Coordinate Reference System - + - + - + Curve - + - + - + Cylindrical Coordinate System - + - + - + Dynamic Feature - + - + - + Dynamic Feature Collection - + - + - + Ellipsoidal Coordinate System - + - + - + Engineering Coordinate Reference System - + - + - + Engineering Datum - + - + - + Geocentric Coordinate Reference System - + - + - + Geodetic Coordinate Reference System - + - + - + Geodetic Datum - + - + - + Geographic Coordinate Reference System - + - + - + Grid - + - + - + Grid Coverage - + - + - + Image Coordinate Reference System - + - + - + Image Datum - + - + - + Line String - + - + - + Linear Coordinate System - + - + - + Multi Curve Coverage - + - + - - + + - + - + Multi Point Coverage - + - + - - + + - + - - + + - + - - + + - + - + Multi Solid Coverage - + - + - - + + - + - + Multi Surface Coverage - + - + - + Oblique Cartesian Coordinate System - + - + - + Orientable Curve - + - + - + Orientable Surface - + - + - + + Plot + + + + + + + + Point - + - + - + Polar Coordinate System - + - + - + Polygon - + - + - + Polyhedral Surface - + - + - + Rectified Grid - + - + - + Rectified Grid Coverage - + + + + + + + + + Replicate Plot + + + + + + + + + ReplicateTransect + - + - + Single Coordinate Reference System - + - + - + Solid - + - + - + Spatial Continuous Coverage - + - + - + Spatial Coordinate Reference System - + - + - + Spatial Coordinate System - + - + - + Spatial Coverage - + - + - + Spatial Curve - + - + - + Spatial Datum - + - + - + Spatial Discreet Coverage - + - + - + Spatial Entity In the Geography Markup Language 3.2.1 Schema, which models spatial concepts, the AbstractTopology type and its subclasses have been omitted as a subclass of this SpatialEntity class because OBOE describes topologies as a Characteritic of multiple Entities using the Relationship characteristic. @@ -567,166 +550,152 @@ - + - + Spatial Feature - + GML 3.2.1 defines a subclass of AbstractFeature called Observation with a subclass of DirectedObservation. This my conflict with the OBOE concept of Observation. For this reason, these two subclasses are not currently included under SpatialFeature, and this issue needs to be reconciled. - + - + Spatial Geometric Aggregate - + - + - + Spatial Geometric Complex - + - + - + Spatial Geometric Primitive - + - + - + Spatial Geometry - + - + - + Spatial Implicit Geometry - + - + - + Spatial Solid - + - + - + Spatial Surface - + - + - + Spherical Coordinate System - + - + - + Surface - + - + - + + Transect + + + + + + + + User Defined Coordinate System - + - + - + Vertical Coordinate Reference System - + - + - + Vertical Coordinate System - + - + - + Vertical Datum - - - - - - - Within - A spatial relationship where an Entity in an Ebservation is within another Entity. - + - - - Adjacent - A spatial relationship where an Entity in an Ebservation is adjacent to another Entity. - - - - - - Overlaps - A spatial relationship where an Entity in an Observation overlaps another Entity. - - - - + - - Spatial Relationship - A relationship between Entities within Observations that has a spatial component (within, adjacent, etc.) - + + Waypoint + + waypoint is usually defined by a latitude-longitude pair. It often defines the nominal location of a station or sampling place. - - + diff --git a/metacat-index/src/main/resources/ontologies/oboe-standards.owl b/metacat-index/src/main/resources/ontologies/oboe-standards.owl old mode 100644 new mode 100755 index a6cc77a8f..c3186bb69 --- a/metacat-index/src/main/resources/ontologies/oboe-standards.owl +++ b/metacat-index/src/main/resources/ontologies/oboe-standards.owl @@ -9,29 +9,31 @@ - - + + + + ]> - - + oboe-standards - Copyright (c) 2006-2011 The Regents of the University of California. All rights reserved. This work is licensed under the Creative Commons Attribution 3.0 Unported License. To view a copy of this license, visit http://creativecommons.org/licenses/by/3.0/ or send a letter to Creative Commons, 444 Castro Street, Suite 900, Mountain View, California, 94041, USA. - Version 1.0 + Copyright (c) 2006-2016 The Regents of the University of California. All rights reserved. This work is licensed under the Creative Commons Attribution 3.0 Unported License. To view a copy of this license, visit http://creativecommons.org/licenses/by/3.0/ or send a letter to Creative Commons, 444 Castro Street, Suite 900, Mountain View, California, 94041, USA. + Version 1.2 This ontology contains terms for measurement standards that are common across OBOE extensions including units of measure and indices. The unit descriptions (comments) were adapted from various sources, including NIST. - - + + @@ -69,9 +71,9 @@ - + - + @@ -125,14 +127,14 @@ - + - + - + @@ -150,16 +152,16 @@ - + - + - + @@ -176,22 +178,22 @@ - + - + - + - + @@ -214,9 +216,9 @@ - + - + @@ -239,20 +241,20 @@ - + - + - + - + @@ -271,14 +273,14 @@ - + - + - + @@ -307,9 +309,9 @@ - + - + @@ -336,14 +338,14 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + - + @@ -355,7 +357,7 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + @@ -369,9 +371,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -382,7 +384,7 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + @@ -405,9 +407,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -461,9 +463,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -474,7 +476,7 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + @@ -486,16 +488,16 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + - + - + @@ -518,9 +520,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -543,20 +545,20 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + - + - + @@ -575,9 +577,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -599,14 +601,14 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + - + @@ -618,7 +620,7 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + @@ -631,9 +633,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -655,15 +657,15 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - - + + - + @@ -692,9 +694,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -721,7 +723,7 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + @@ -735,21 +737,21 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + - + - + - + @@ -778,9 +780,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -791,13 +793,13 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + - + @@ -810,9 +812,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -834,9 +836,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -853,22 +855,22 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + - + - + - + @@ -890,9 +892,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -915,14 +917,14 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + - + @@ -934,7 +936,7 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + @@ -947,9 +949,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -971,9 +973,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -990,22 +992,22 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + - + - + - + @@ -1027,14 +1029,14 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + - + @@ -1052,16 +1054,16 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + - + - + @@ -1083,9 +1085,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -1102,27 +1104,27 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + - + - + - + - + @@ -1134,7 +1136,7 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + @@ -1147,9 +1149,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -1171,9 +1173,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -1195,14 +1197,14 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + - + @@ -1214,7 +1216,7 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + @@ -1227,9 +1229,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -1251,14 +1253,14 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + - + @@ -1270,7 +1272,7 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + @@ -1283,9 +1285,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -1307,9 +1309,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -1326,22 +1328,22 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + - + - + - + @@ -1363,9 +1365,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -1382,22 +1384,22 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + - + - + - + @@ -1419,14 +1421,14 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + - + @@ -1438,7 +1440,7 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + @@ -1451,9 +1453,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -1475,9 +1477,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -1499,14 +1501,14 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + - + @@ -1518,7 +1520,7 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + @@ -1531,9 +1533,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -1560,21 +1562,21 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + - + - + - + @@ -1592,16 +1594,16 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + - + - + @@ -1655,9 +1657,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -1668,7 +1670,7 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + @@ -1680,16 +1682,16 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + - + - + @@ -1711,9 +1713,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -1724,13 +1726,13 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + - + @@ -1743,9 +1745,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -1767,9 +1769,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -1796,7 +1798,7 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + @@ -1808,7 +1810,7 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + @@ -1823,9 +1825,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -1836,7 +1838,7 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + @@ -1850,7 +1852,7 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + @@ -1879,9 +1881,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -1892,7 +1894,7 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + @@ -1928,16 +1930,16 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + - + - + @@ -1948,13 +1950,13 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + - + @@ -1967,9 +1969,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -1990,7 +1992,7 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + @@ -2003,9 +2005,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -2027,9 +2029,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -2046,22 +2048,22 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + - + - + - + @@ -2083,14 +2085,14 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + - + @@ -2102,7 +2104,7 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + @@ -2115,10 +2117,10 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - - + + @@ -2141,29 +2143,29 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + - + - + - + - + @@ -2185,14 +2187,14 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + - + @@ -2210,16 +2212,16 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + - + - + @@ -2240,7 +2242,7 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + @@ -2253,9 +2255,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -2277,9 +2279,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -2290,13 +2292,13 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + - + @@ -2309,9 +2311,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -2333,9 +2335,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -2346,7 +2348,7 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + @@ -2369,14 +2371,14 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + - + @@ -2394,24 +2396,17 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + - - - - - - - - + - + @@ -2433,9 +2428,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -2457,14 +2452,14 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + - + @@ -2476,7 +2471,7 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + @@ -2513,14 +2508,14 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + - + @@ -2538,7 +2533,7 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + @@ -2569,14 +2564,14 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + - + @@ -2610,7 +2605,7 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + @@ -2625,9 +2620,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -2638,7 +2633,7 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + @@ -2668,7 +2663,7 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + @@ -2680,7 +2675,7 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + @@ -2701,14 +2696,14 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + - + @@ -2737,10 +2732,10 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - - + + @@ -2751,7 +2746,7 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + @@ -2774,9 +2769,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -2798,9 +2793,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -2827,27 +2822,27 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + - + - + - + - + @@ -2866,9 +2861,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -2890,9 +2885,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -2914,9 +2909,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -2933,22 +2928,22 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + - + - + - + @@ -2959,7 +2954,7 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + @@ -2987,7 +2982,7 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + @@ -3003,9 +2998,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -3027,9 +3022,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -3040,7 +3035,7 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + @@ -3052,16 +3047,16 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + - + - + @@ -3082,7 +3077,7 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + @@ -3095,9 +3090,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -3124,7 +3119,7 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + @@ -3144,16 +3139,16 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + - + - + @@ -3164,7 +3159,7 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + @@ -3176,16 +3171,16 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + - + - + @@ -3207,9 +3202,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -3231,14 +3226,14 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + - + @@ -3256,16 +3251,16 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + - + - + @@ -3276,7 +3271,7 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + @@ -3299,9 +3294,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -3323,14 +3318,14 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + - + @@ -3342,7 +3337,7 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + @@ -3355,9 +3350,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -3379,9 +3374,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -3398,22 +3393,22 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + - + - + - + @@ -3435,14 +3430,14 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + - + @@ -3471,14 +3466,14 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + - + @@ -3512,7 +3507,7 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + @@ -3527,9 +3522,9 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) + - + @@ -3556,16 +3551,16 @@ Common derivatives are millibar (meteorology) and decibar (oceanography) - + - + - + @@ -3597,9 +3592,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -3621,14 +3616,14 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + - + @@ -3662,7 +3657,7 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + @@ -3677,9 +3672,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -3696,7 +3691,7 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + @@ -3726,21 +3721,21 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + - + - + @@ -3752,7 +3747,7 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + @@ -3789,20 +3784,20 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + - + - + @@ -3821,14 +3816,14 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + - + @@ -3857,9 +3852,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -3881,9 +3876,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -3904,7 +3899,7 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + @@ -3917,9 +3912,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -3930,7 +3925,7 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + @@ -3942,16 +3937,16 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + - + @@ -3974,9 +3969,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -3998,9 +3993,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -4022,9 +4017,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -4046,20 +4041,20 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + - + - + @@ -4078,9 +4073,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -4102,14 +4097,14 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + - + @@ -4121,7 +4116,7 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + @@ -4134,9 +4129,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -4158,9 +4153,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -4182,14 +4177,14 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + - + @@ -4218,9 +4213,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -4231,7 +4226,7 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + @@ -4243,16 +4238,16 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + - + @@ -4275,9 +4270,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -4288,13 +4283,13 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -4307,9 +4302,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -4331,14 +4326,14 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + - + @@ -4356,7 +4351,7 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + @@ -4387,9 +4382,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -4422,7 +4417,7 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + @@ -4436,16 +4431,16 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + - + @@ -4462,7 +4457,7 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + @@ -4476,7 +4471,7 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + @@ -4488,7 +4483,7 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + @@ -4519,14 +4514,14 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + - + @@ -4538,7 +4533,7 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + @@ -4558,7 +4553,7 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + @@ -4595,9 +4590,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -4614,22 +4609,22 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + - + - + @@ -4650,7 +4645,7 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + @@ -4663,9 +4658,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -4676,7 +4671,7 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + @@ -4699,9 +4694,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -4718,7 +4713,7 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + @@ -4732,7 +4727,7 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + @@ -4756,9 +4751,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -4780,9 +4775,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -4803,7 +4798,7 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + @@ -4816,9 +4811,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -4829,7 +4824,7 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + @@ -4841,16 +4836,16 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + - + @@ -4872,9 +4867,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -4896,9 +4891,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -4920,14 +4915,14 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + - + @@ -4945,16 +4940,16 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + - + @@ -4976,9 +4971,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -5000,9 +4995,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -5013,13 +5008,13 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -5032,9 +5027,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -5056,9 +5051,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -5085,7 +5080,7 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + @@ -5105,16 +5100,16 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + - + @@ -5147,7 +5142,7 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + @@ -5161,16 +5156,16 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + - + @@ -5192,14 +5187,14 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + - + @@ -5227,7 +5222,7 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + @@ -5248,9 +5243,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -5267,22 +5262,22 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + - + - + @@ -5304,9 +5299,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -5323,22 +5318,22 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + - + - + @@ -5360,9 +5355,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -5384,14 +5379,14 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + - + @@ -5403,7 +5398,7 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + @@ -5416,9 +5411,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -5440,20 +5435,20 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + - + - + @@ -5472,9 +5467,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -5491,7 +5486,7 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + @@ -5521,7 +5516,7 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + @@ -5533,7 +5528,7 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + @@ -5550,9 +5545,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -5574,14 +5569,14 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + - + @@ -5593,7 +5588,7 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + @@ -5606,9 +5601,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -5630,14 +5625,14 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + - + @@ -5649,7 +5644,7 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + @@ -5662,9 +5657,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -5686,14 +5681,14 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + - + @@ -5705,7 +5700,7 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + @@ -5718,9 +5713,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -5742,14 +5737,14 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + - + @@ -5761,7 +5756,7 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + @@ -5774,9 +5769,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -5798,9 +5793,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -5811,13 +5806,13 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -5830,9 +5825,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -5854,9 +5849,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -5873,22 +5868,22 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + - + - + @@ -5910,9 +5905,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -5923,7 +5918,7 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + @@ -5935,7 +5930,7 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + @@ -5966,9 +5961,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -5990,9 +5985,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -6009,7 +6004,7 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + @@ -6029,7 +6024,7 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + @@ -6059,7 +6054,7 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + The pascal (symbol: Pa) is the SI derived unit of pressure. It is a measure of force per unit area, and defined as one newton per sqare meter. @@ -6068,9 +6063,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -6093,9 +6088,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -6149,20 +6144,20 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + - + - + @@ -6181,9 +6176,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -6205,20 +6200,20 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + - + - + @@ -6237,19 +6232,19 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + Practical Salinity Unit - + At the simplest level, salinity is the total amount of dissolved material in grams in one kilogram of sea water. So it is typically thought of as a dimensionless quantity with no units. It is described here as an Index. For more information, see current physical oceanographic references, such as http://oceanworld.tamu.edu/resources/ocng_textbook/chapter06/chapter06_01.htm - + - + @@ -6303,9 +6298,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -6322,22 +6317,22 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + - + - + @@ -6359,9 +6354,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -6372,7 +6367,7 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + @@ -6395,9 +6390,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -6419,9 +6414,9 @@ It is named in honor of Albert Einstein, who in a 1905 paper explained the photo - + - + @@ -6448,9 +6443,9 @@ a siemens is Ampere/volt - + - + @@ -6483,7 +6478,7 @@ a siemens is Ampere/volt - + @@ -6497,7 +6492,7 @@ a siemens is Ampere/volt - + Conductivity is the reciprocal (inverse) of electrical resistivity, ρ, and has the SI units of siemens per metre (S·m-1). "siemens" is used for both singular and plural. @@ -6505,9 +6500,9 @@ a siemens is Ampere/volt - + - + @@ -6529,14 +6524,14 @@ a siemens is Ampere/volt - + - + - + @@ -6548,7 +6543,7 @@ a siemens is Ampere/volt - + @@ -6561,9 +6556,9 @@ a siemens is Ampere/volt - + - + @@ -6585,9 +6580,9 @@ a siemens is Ampere/volt - + - + @@ -6598,7 +6593,7 @@ a siemens is Ampere/volt - + @@ -6610,16 +6605,16 @@ a siemens is Ampere/volt - + - + - + @@ -6642,9 +6637,9 @@ a siemens is Ampere/volt - + - + @@ -6666,9 +6661,9 @@ a siemens is Ampere/volt - + - + @@ -6690,9 +6685,9 @@ a siemens is Ampere/volt - + - + @@ -6714,9 +6709,9 @@ a siemens is Ampere/volt - + - + @@ -6738,9 +6733,9 @@ a siemens is Ampere/volt - + - + @@ -6757,22 +6752,22 @@ a siemens is Ampere/volt - + - + - + - + @@ -6793,7 +6788,7 @@ a siemens is Ampere/volt - + @@ -6806,9 +6801,9 @@ a siemens is Ampere/volt - + - + @@ -6819,7 +6814,7 @@ a siemens is Ampere/volt - + @@ -6831,7 +6826,7 @@ a siemens is Ampere/volt - + diff --git a/metacat-index/src/main/resources/ontologies/oboe-taxa.owl b/metacat-index/src/main/resources/ontologies/oboe-taxa.owl old mode 100644 new mode 100755 diff --git a/metacat-index/src/main/resources/ontologies/oboe-temporal.owl b/metacat-index/src/main/resources/ontologies/oboe-temporal.owl old mode 100644 new mode 100755 index 3707727be..0a72e493f --- a/metacat-index/src/main/resources/ontologies/oboe-temporal.owl +++ b/metacat-index/src/main/resources/ontologies/oboe-temporal.owl @@ -2,33 +2,35 @@ - - - - - - - + + + + + + + + + + ]> - - + OBOE Temporal - Version 1.0 + Version 1.1 Copyright (c) 2006-2011 The Regents of the University of California. All rights reserved. This work is licensed under the Creative Commons Attribution 3.0 Unported License. To view a copy of this license, visit http://creativecommons.org/licenses/by/3.0/ or send a letter to Creative Commons, 444 Castro Street, Suite 900, Mountain View, California, 94041, USA. This ontology terms relating to temporal concepts that are common across OBOE extensions. The terms are derived from the International Standards Organization (ISO) geospatial standards in the 19100 series and from the Geography Markup Language version 3.2.1. - + @@ -83,136 +85,136 @@ --> - + - + Temporal Entity - + - + Time Calendar - + - + - + Time Clock - + - + - + Time Coordinate Reference System - + - + - + Time Datum - + - + - + Time Geometric Primitive - + - + - + Time Instant - + - + - + Time Ordinal Reference System - + - + - + Time Period - + - + - + Time Primitive - + - + - + Time Reference System - + - + - + Temporal Relationship - + - + After A temporal relationship where an Entity is observed after another Entity. - + - + - + Before A temporal relationship where an Entity is observed prior to another Entity. - + - + - + During A temporal relationship where an Entity is observed concurrently with another Entity. - + diff --git a/metacat-index/src/main/resources/ontologies/oboe.owl b/metacat-index/src/main/resources/ontologies/oboe.owl old mode 100644 new mode 100755 index 088e796fe..e7fccc304 --- a/metacat-index/src/main/resources/ontologies/oboe.owl +++ b/metacat-index/src/main/resources/ontologies/oboe.owl @@ -9,56 +9,33 @@ - - - - - - - - - - - + + + + ]> - - + oboe - Version 1.0 - Copyright (c) 2006-2011 The Regents of the University of California. All rights reserved. This work is licensed under the Creative Commons Attribution 3.0 Unported License. To view a copy of this license, visit http://creativecommons.org/licenses/by/3.0/ or send a letter to Creative Commons, 444 Castro Street, Suite 900, Mountain View, California, 94041, USA. + Version 1.2 + Copyright (c) 2006-2016 The Regents of the University of California. All rights reserved. This work is licensed under the Creative Commons Attribution 3.0 Unported License. To view a copy of this license, visit http://creativecommons.org/licenses/by/3.0/ or send a letter to Creative Commons, 444 Castro Street, Suite 900, Mountain View, California, 94041, USA. This ontology contains general terms that are common across OBOE extensions. It extends the OBOE Core model by importing ontologies specific to measurement characteristics, measurement standards, spatial entities, temporal entities, biological taxa, and anatomical entities. - - - - - - - - - - - + + + From de08a2506aca4bb52e8a122a4fe5b5b6c3569447 Mon Sep 17 00:00:00 2001 From: Jing Tao Date: Fri, 3 Sep 2021 12:43:00 -0700 Subject: [PATCH 39/45] Expand the attributes part from datatable to any entities. --- .../resources/application-context-eml-base.xml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/metacat-index/src/main/resources/application-context-eml-base.xml b/metacat-index/src/main/resources/application-context-eml-base.xml index 8cfb453eb..b8b567b52 100644 --- a/metacat-index/src/main/resources/application-context-eml-base.xml +++ b/metacat-index/src/main/resources/application-context-eml-base.xml @@ -424,7 +424,7 @@ + value='//attributeList/attribute/attributeName/text()' /> @@ -432,7 +432,7 @@ + value='//attributeList/attribute/attributeLabel/text()' /> @@ -440,7 +440,7 @@ + value='//attributeList/attribute/attributeDefinition/text()' /> @@ -448,7 +448,7 @@ + value='//attributeList/attribute//standardUnit/text() | //attributeList/attribute//customUnit/text()' /> @@ -461,7 +461,7 @@ @@ -492,7 +492,7 @@ + value='//attributeList/attribute/attributeName/text()' /> @@ -500,7 +500,7 @@ + value='//attributeList/attribute/attributeLabel/text()' /> @@ -508,7 +508,7 @@ + value='//attributeList/attribute/attributeDefinition/text()' /> @@ -516,7 +516,7 @@ + value='//attributeList/attribute//standardUnit/text() | //attributeList/attribute//customUnit/text()' /> From aa72528fd3fd9e8f0e17050b15a5263e21a1c9a0 Mon Sep 17 00:00:00 2001 From: Jing Tao Date: Fri, 3 Sep 2021 17:13:38 -0700 Subject: [PATCH 40/45] Remove the dependency on commons-httpclient-3.1.jar file --- pom.xml | 5 -- .../metacat/annotation/OrcidService.java | 41 ++++++++++++---- ...StreamingMultipartRequestResolverTest.java | 49 +++++++++++-------- 3 files changed, 59 insertions(+), 36 deletions(-) diff --git a/pom.xml b/pom.xml index 243c31c36..068b72505 100644 --- a/pom.xml +++ b/pom.xml @@ -313,11 +313,6 @@ 2.7 - commons-httpclient - commons-httpclient - 3.1 - - org.dspace oaicat 1.5.48 diff --git a/src/edu/ucsb/nceas/metacat/annotation/OrcidService.java b/src/edu/ucsb/nceas/metacat/annotation/OrcidService.java index d467ad24d..46837315f 100644 --- a/src/edu/ucsb/nceas/metacat/annotation/OrcidService.java +++ b/src/edu/ucsb/nceas/metacat/annotation/OrcidService.java @@ -1,5 +1,6 @@ package edu.ucsb.nceas.metacat.annotation; +import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.StringReader; @@ -8,10 +9,11 @@ import java.util.ArrayList; import java.util.List; - -import org.apache.commons.httpclient.HttpClient; -import org.apache.commons.httpclient.HttpMethod; -import org.apache.commons.httpclient.methods.GetMethod; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -28,6 +30,9 @@ public class OrcidService { //private static final String REST_URL = "http://pub.sandbox.orcid.org/v1.1/search/orcid-bio"; private static final String REST_URL = "https://pub.orcid.org/v2.0/search"; + + private static RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(5 * 1000).build(); + private static CloseableHttpClient client = HttpClientBuilder.create().setDefaultRequestConfig(requestConfig).build(); /** @@ -41,6 +46,8 @@ public class OrcidService { public static String lookupOrcid(String text, String surName, List givenNames, List otherNames) { String url = null; + CloseableHttpResponse response = null; + InputStream is = null; try { @@ -70,12 +77,10 @@ public static String lookupOrcid(String text, String surName, List given urlParameters = URLEncoder.encode(urlParameters, "UTF-8"); url = REST_URL + "?q=" + urlParameters + "&rows=1"; - URL restURL = new URL(url); - HttpClient client = new HttpClient(); - HttpMethod method = new GetMethod(url); - method.addRequestHeader("Accept", "application/orcid+xml"); - client.executeMethod(method); - InputStream is = method.getResponseBodyAsStream(); + HttpGet method = new HttpGet(url); + method.addHeader("Accept", "application/orcid+xml"); + response = client.execute(method); + is = response.getEntity().getContent(); //InputStream is = restURL.openStream(); String results = IOUtils.toString(is, "UTF-8"); @@ -89,6 +94,22 @@ public static String lookupOrcid(String text, String surName, List given } } catch (Exception e) { logMetacat.error("Could not lookup ORCID using: " + url, e); + } finally { + if (response != null) { + try { + response.close(); + } catch (IOException ee) { + logMetacat.warn("OrcidServic.lookupOrcid - could not close the http response from the ORCID service since " + ee.getMessage()); + } + + } + if (is != null) { + try { + is.close(); + } catch (IOException ee) { + logMetacat.warn("OrcidServic.lookupOrcid - could not close the input stream object from the http response " + ee.getMessage()); + } + } } return null; diff --git a/test/edu/ucsb/nceas/metacat/restservice/multipart/StreamingMultipartRequestResolverTest.java b/test/edu/ucsb/nceas/metacat/restservice/multipart/StreamingMultipartRequestResolverTest.java index ca493f9b9..3b590d39b 100644 --- a/test/edu/ucsb/nceas/metacat/restservice/multipart/StreamingMultipartRequestResolverTest.java +++ b/test/edu/ucsb/nceas/metacat/restservice/multipart/StreamingMultipartRequestResolverTest.java @@ -33,12 +33,12 @@ import javax.servlet.ServletInputStream; import javax.servlet.http.HttpServletRequest; -import org.apache.commons.httpclient.methods.PostMethod; -import org.apache.commons.httpclient.methods.multipart.ByteArrayPartSource; -import org.apache.commons.httpclient.methods.multipart.FilePart; -import org.apache.commons.httpclient.methods.multipart.MultipartRequestEntity; -import org.apache.commons.httpclient.methods.multipart.Part; -import org.apache.commons.httpclient.methods.multipart.StringPart; +import org.apache.http.HttpEntity; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.mime.HttpMultipartMode; +import org.apache.http.entity.mime.MultipartEntityBuilder; +import org.apache.http.entity.mime.content.ByteArrayBody; +import org.apache.http.entity.mime.content.StringBody; import org.dataone.configuration.Settings; import org.dataone.mimemultipart.MultipartRequest; import org.dataone.service.types.v1.Checksum; @@ -53,7 +53,6 @@ import org.mockito.Mockito; import edu.ucsb.nceas.metacat.dataone.D1NodeServiceTest; -import edu.ucsb.nceas.metacat.properties.PropertyService; import junit.framework.Test; import junit.framework.TestSuite; @@ -126,19 +125,23 @@ public void testV2ResolveMultipart() throws Exception { TypeMarshaller.marshalTypeToOutputStream(sysmeta, sysOutput); byte[] sysContent = sysOutput.toByteArray(); - // Create part & entity from resource - Part[] parts = new Part[] { new StringPart("pid", guid.getValue()), new FilePart(StreamingMultipartRequestResolver.SYSMETA, new ByteArrayPartSource("sysmetametadata.xml", sysContent)), - new FilePart("object", new ByteArrayPartSource(objectFile, fileContent)) }; - MultipartRequestEntity multipartRequestEntity = - new MultipartRequestEntity(parts, new PostMethod().getParams()); + MultipartEntityBuilder builder = MultipartEntityBuilder.create(); + builder.setMode(HttpMultipartMode.BROWSER_COMPATIBLE); + StringBody pidBody = new StringBody(guid.getValue(), ContentType.MULTIPART_FORM_DATA); + builder.addPart("pid", pidBody); + ByteArrayBody sysmetaBody = new ByteArrayBody(sysContent, "sysmetametadata.xml"); + builder.addPart(StreamingMultipartRequestResolver.SYSMETA, sysmetaBody); + ByteArrayBody objectBody = new ByteArrayBody(fileContent, objectFile); + builder.addPart("object", objectBody); + HttpEntity entity = builder.build(); // Serialize request body ByteArrayOutputStream requestContent = new ByteArrayOutputStream(); - multipartRequestEntity.writeRequest(requestContent); + entity.writeTo(requestContent); ByteArrayInputStream requestInput = new ByteArrayInputStream(requestContent.toByteArray()); ServletInputStream objectInputStream = new WrappingServletInputStream(requestInput); HttpServletRequest request = Mockito.mock(HttpServletRequest.class); Mockito.when(request.getMethod()).thenReturn("post"); - Mockito.when(request.getContentType()).thenReturn(multipartRequestEntity.getContentType()); + Mockito.when(request.getContentType()).thenReturn(entity.getContentType().getValue()); Mockito.when(request.getInputStream()).thenReturn(objectInputStream); StreamingMultipartRequestResolver resolver = new StreamingMultipartRequestResolver("build", 10000000); MultipartRequest result = resolver.resolveMultipart(request); @@ -200,19 +203,23 @@ public void testV1ResolveMultipart() throws Exception { TypeMarshaller.marshalTypeToOutputStream(sysmeta, sysOutput); byte[] sysContent = sysOutput.toByteArray(); - // Create part & entity from resource - Part[] parts = new Part[] { new StringPart("pid", guid.getValue()), new FilePart(StreamingMultipartRequestResolver.SYSMETA, new ByteArrayPartSource("sysmetametadata.xml", sysContent)), - new FilePart("object", new ByteArrayPartSource(objectFile, fileContent)) }; - MultipartRequestEntity multipartRequestEntity = - new MultipartRequestEntity(parts, new PostMethod().getParams()); + MultipartEntityBuilder builder = MultipartEntityBuilder.create(); + builder.setMode(HttpMultipartMode.BROWSER_COMPATIBLE); + StringBody pidBody = new StringBody(guid.getValue(), ContentType.MULTIPART_FORM_DATA); + builder.addPart("pid", pidBody); + ByteArrayBody sysmetaBody = new ByteArrayBody(sysContent, "sysmetametadata.xml"); + builder.addPart(StreamingMultipartRequestResolver.SYSMETA, sysmetaBody); + ByteArrayBody objectBody = new ByteArrayBody(fileContent, objectFile); + builder.addPart("object", objectBody); + HttpEntity entity = builder.build(); // Serialize request body ByteArrayOutputStream requestContent = new ByteArrayOutputStream(); - multipartRequestEntity.writeRequest(requestContent); + entity.writeTo(requestContent); ByteArrayInputStream requestInput = new ByteArrayInputStream(requestContent.toByteArray()); ServletInputStream objectInputStream = new WrappingServletInputStream(requestInput); HttpServletRequest request = Mockito.mock(HttpServletRequest.class); Mockito.when(request.getMethod()).thenReturn("post"); - Mockito.when(request.getContentType()).thenReturn(multipartRequestEntity.getContentType()); + Mockito.when(request.getContentType()).thenReturn(entity.getContentType().getValue()); Mockito.when(request.getInputStream()).thenReturn(objectInputStream); StreamingMultipartRequestResolver resolver = new StreamingMultipartRequestResolver("build", 10000000); MultipartRequest result = resolver.resolveMultipart(request); From f719e74612bdc75ea243165d511e367b4b69c2a9 Mon Sep 17 00:00:00 2001 From: Jing Tao Date: Tue, 7 Sep 2021 12:21:18 -0700 Subject: [PATCH 41/45] Add a map for the eml-2.2.0 namespace. Ref: https://github.com/NCEAS/metacat/issues/1522 --- lib/style/skins/default/default.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/style/skins/default/default.xml b/lib/style/skins/default/default.xml index 6a8263d8b..c42d7b810 100755 --- a/lib/style/skins/default/default.xml +++ b/lib/style/skins/default/default.xml @@ -31,6 +31,11 @@ /style/skins/default/default.xsl + + + /style/skins/default/default.xsl + + /style/skins/default/default.xsl From 9141b2672363bb54e61d566fb16b7d2df5a6b100 Mon Sep 17 00:00:00 2001 From: Jing Tao Date: Tue, 7 Sep 2021 14:16:53 -0700 Subject: [PATCH 42/45] Change the speedbagit version from snapshot to 1.0.0. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 068b72505..d0db34c82 100644 --- a/pom.xml +++ b/pom.xml @@ -437,7 +437,7 @@ org.dataone speedbagit - 1.0-SNAPSHOT + 1.0.0 jar From dbc1a44a07c66a3e742811c14704e2b943a67dbf Mon Sep 17 00:00:00 2001 From: Jing Tao Date: Wed, 8 Sep 2021 14:51:40 -0700 Subject: [PATCH 43/45] Update the version to 2.15.1. --- build.properties | 2 +- lib/metacat.properties | 3 ++- metacat-common/pom.xml | 2 +- metacat-index/pom.xml | 4 ++-- pom.xml | 4 ++-- src/loaddtdschema-postgres.sql | 2 +- src/upgrade-db-to-2.15.1-postgres.sql | 16 ++++++++++++++++ src/xmltables-postgres.sql | 2 ++ 8 files changed, 27 insertions(+), 8 deletions(-) create mode 100644 src/upgrade-db-to-2.15.1-postgres.sql diff --git a/build.properties b/build.properties index 6de591ce3..417758897 100755 --- a/build.properties +++ b/build.properties @@ -2,7 +2,7 @@ #Version of this build. This needs to be a dotted numeric version. For #instance 1.9.1 is okay. 1.9.1_rc1 is not. -metacat.version=2.15.0 +metacat.version=2.15.1 #This is for packaging purposes. leave it blank for final production release. metacat.releaseCandidate= diff --git a/lib/metacat.properties b/lib/metacat.properties index 35a0a3560..66ddd6188 100755 --- a/lib/metacat.properties +++ b/lib/metacat.properties @@ -34,7 +34,7 @@ server.internalPort=80 ############### Application Values ############ ## one of the few places where we use ANT tokens -application.metacatVersion=2.15.0 +application.metacatVersion=2.15.1 application.metacatReleaseInfo=-1 application.readOnlyMode=false @@ -134,6 +134,7 @@ database.upgradeVersion.2.13.0=upgrade-db-to-2.13.0 database.upgradeVersion.2.14.0=upgrade-db-to-2.14.0 database.upgradeVersion.2.14.1=upgrade-db-to-2.14.1 database.upgradeVersion.2.15.0=upgrade-db-to-2.15.0 +database.upgradeVersion.2.15.1=upgrade-db-to-2.15.1 ## for running java-based utilities database.upgradeUtility.1.5.0=edu.ucsb.nceas.metacat.admin.upgrade.Upgrade1_5_0 diff --git a/metacat-common/pom.xml b/metacat-common/pom.xml index 798eb7f35..814e1b1f9 100644 --- a/metacat-common/pom.xml +++ b/metacat-common/pom.xml @@ -4,7 +4,7 @@ edu.ucsb.nceas.metacat.common metacat-common jar - 2.15.0 + 2.15.1 metacat-common http://maven.apache.org diff --git a/metacat-index/pom.xml b/metacat-index/pom.xml index 8206535e6..b5583b81e 100644 --- a/metacat-index/pom.xml +++ b/metacat-index/pom.xml @@ -4,13 +4,13 @@ edu.ucsb.nceas.metacat.index metacat-index war - 2.15.0 + 2.15.1 metacat-index http://maven.apache.org 2.3.14 - 2.15.0 + 2.15.1 diff --git a/pom.xml b/pom.xml index d0db34c82..1301b63e8 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 org.ecoinformatics metacat - 2.15.0 + 2.15.1 metacat war http://maven.apache.org @@ -12,7 +12,7 @@ UTF-8 2.3.1 2.3.2 - 2.15.0 + 2.15.1 diff --git a/src/loaddtdschema-postgres.sql b/src/loaddtdschema-postgres.sql index 2a4fd5e3f..8811b7bbe 100755 --- a/src/loaddtdschema-postgres.sql +++ b/src/loaddtdschema-postgres.sql @@ -217,4 +217,4 @@ INSERT INTO xml_catalog (entry_type, public_id, system_id) SELECT 'Schema', 'htt INSERT INTO xml_catalog (entry_type, public_id, format_id) SELECT 'NonXML', 'science-on-schema.org/Dataset;ld+json', 'science-on-schema.org/Dataset;ld+json' WHERE NOT EXISTS (SELECT * FROM xml_catalog WHERE public_id='science-on-schema.org/Dataset;ld+json'); INSERT INTO db_version (version, status, date_created) - VALUES ('2.15.0',1,CURRENT_DATE); + VALUES ('2.15.1',1,CURRENT_DATE); diff --git a/src/upgrade-db-to-2.15.1-postgres.sql b/src/upgrade-db-to-2.15.1-postgres.sql new file mode 100644 index 000000000..da1b61471 --- /dev/null +++ b/src/upgrade-db-to-2.15.1-postgres.sql @@ -0,0 +1,16 @@ +/* + * Ensure xml_catalog sequence is at table max + */ + +SELECT setval('xml_catalog_id_seq', (SELECT max(catalog_id) from xml_catalog)); + +CREATE INDEX systemMetadata_object_format on systemMetadata(object_format); +CREATE INDEX systemMetadata_archived on systemMetadata(archived); + +/* + * update the database version + */ +UPDATE db_version SET status=0; + +INSERT INTO db_version (version, status, date_created) + VALUES ('2.15.1', 1, CURRENT_DATE); diff --git a/src/xmltables-postgres.sql b/src/xmltables-postgres.sql index 399789ac9..64e98940f 100755 --- a/src/xmltables-postgres.sql +++ b/src/xmltables-postgres.sql @@ -325,6 +325,8 @@ CREATE TABLE systemMetadata ( CREATE INDEX systemMetadata_series_id on systemMetadata(series_id); CREATE INDEX systemMetadata_date_uploaded on systemMetadata(date_uploaded); CREATE INDEX systemMetadata_date_modified on systemMetadata(date_modified); +CREATE INDEX systemMetadata_object_format on systemMetadata(object_format); +CREATE INDEX systemMetadata_archived on systemMetadata(archived); /* * Table used to store the properties for media types. They are part of the system metadata. But a media type From 0f4912fcc785de30e63940094d4ba63ffe6c65fd Mon Sep 17 00:00:00 2001 From: Jing Tao Date: Thu, 9 Sep 2021 10:39:21 -0700 Subject: [PATCH 44/45] Change the version of MetacatUI to 2.16.0 --- build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.xml b/build.xml index 5d6002b65..70dc9b547 100755 --- a/build.xml +++ b/build.xml @@ -32,7 +32,7 @@ - + From 67bf23a6929c467b2e111f11b704423efd761d2a Mon Sep 17 00:00:00 2001 From: Jing Tao Date: Fri, 10 Sep 2021 11:34:03 -0700 Subject: [PATCH 45/45] Add the release node for 2.15.1 --- README.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1075b831b..f00cfaf39 100755 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Metacat: Data Preservation and Discovery System -Version: 2.15.0 Release +Version: 2.15.1 Release Send feedback and bugs to: metacat-dev@ecoinformatics.org http://github.com/NCEAS/metacat @@ -67,6 +67,19 @@ for the next release. ## Release Notes +### Release Notes for 2.15.1 +New features and bugs fixed in this release: +* Metacat hangs with excessive thread counts +* Support multiple DOI shoulders +* Update EML Semantic Annotation indexing to include and expand property URIs +* MNodeService.getPackage() takes too long for large packages +* Geohashes, text fields not being indexed for schema.org documents +* Expand elements covered by EML's attribute index fields beyond just dataTable +* EML to HTML/PDF is broken in the getPackage() method +* Have MNCore.getCapabilities() report on auth.allowSubmitters parameter setting +* GetPackage API doesn't work from R on Windows +* Add new indexes for column archvied and object_format in the systemmetadata table + ### Release Notes for 2.15.0 New features and bugs fixed in this release: * Support non-XML metadata objects in Metacat