WAT: Duplicated payload metadata values for "Actual-Content-Length" and "Trailing-Slop-Length"

sebastian-nagel · sebastian-nagel · commit 8f4c43f17718 · 2024-12-11T21:05:55.000+01:00
fixes #43 - avoid to add a duplicated "Actual-Content-Length" value - do not add a second value of "Trailing-Slop-Length" if 0
diff --git a/src/main/java/org/archive/resource/arc/ARCResource.java b/src/main/java/org/archive/resource/arc/ARCResource.java
@@ -64,10 +64,12 @@ public ARCResource(MetaData metaData, ResourceContainer container,
 		}
 	}
 
+	@Override
 	public InputStream getInputStream() {
 		return new EOFNotifyingInputStream(digIS, this);
 	}
 
+	@Override
 	public void notifyEOF() throws IOException {
 		metaData.putLong(PAYLOAD_LENGTH, countingIS.getCount());
 		String digString = Base32.encode(digIS.getMessageDigest().digest());
diff --git a/src/main/java/org/archive/resource/http/HTTPHeadersResourceFactory.java b/src/main/java/org/archive/resource/http/HTTPHeadersResourceFactory.java
@@ -31,6 +31,7 @@ public HTTPHeadersResourceFactory(String name, String type) {
 		parser = new HttpHeaderParser();
 	}
 
+	@Override
 	public Resource getResource(InputStream is, MetaData parentMetaData,
 			ResourceContainer container) throws ResourceParseException,
 			IOException {
@@ -40,9 +41,13 @@ public Resource getResource(InputStream is, MetaData parentMetaData,
 			if(headers.isCorrupt()) {
 				parentMetaData.putBoolean(HTTP_HEADERS_CORRUPT, true);
 			}
-			parentMetaData.putLong(PAYLOAD_LENGTH, bytes);
-			
-			parentMetaData.putLong(PAYLOAD_SLOP_BYTES, StreamCopy.readToEOF(is));
+			if (!parentMetaData.has(PAYLOAD_LENGTH) || bytes != parentMetaData.getLong(PAYLOAD_LENGTH)) {
+				parentMetaData.putLong(PAYLOAD_LENGTH, bytes);
+			}
+			long trailingSlopBytes = StreamCopy.readToEOF(is);
+			if (!parentMetaData.has(PAYLOAD_SLOP_BYTES) || trailingSlopBytes > 0) {
+				parentMetaData.putLong(PAYLOAD_SLOP_BYTES, trailingSlopBytes);
+			}
 			if(type != null) {
 				parentMetaData.putString(PAYLOAD_CONTENT_TYPE, type);
 			}
diff --git a/src/main/java/org/archive/resource/warc/WARCResource.java b/src/main/java/org/archive/resource/warc/WARCResource.java
@@ -53,7 +53,7 @@ public WARCResource(MetaData metaData, ResourceContainer container,
 			countingIS = new CountingInputStream(
 					ByteStreams.limit(response, length));
 		} else {
-			throw new ResourceParseException(null);
+			throw new ResourceParseException(new Exception("Zero or negative length: " + length));
 		}
 		try {
 			digIS = new DigestInputStream(countingIS, 
@@ -63,14 +63,18 @@ public WARCResource(MetaData metaData, ResourceContainer container,
 		}
 	}
 
+	@Override
 	public InputStream getInputStream() {
 		return new EOFNotifyingInputStream(digIS, this);
 	}
 
+	@Override
 	public void notifyEOF() throws IOException {
 		String digString = Base32.encode(digIS.getMessageDigest().digest());
 		if(container.isCompressed()) {
-			metaData.putLong(PAYLOAD_LENGTH, countingIS.getCount());
+			if (!metaData.has(PAYLOAD_LENGTH) || countingIS.getCount() != metaData.getLong(PAYLOAD_LENGTH)) {
+				metaData.putLong(PAYLOAD_LENGTH, countingIS.getCount());
+			}
 			metaData.putLong(PAYLOAD_SLOP_BYTES, StreamCopy.readToEOF(response));
 			metaData.putString(PAYLOAD_DIGEST, "sha1:"+digString);
 		} else {
@@ -81,13 +85,17 @@ public void notifyEOF() throws IOException {
 					(PushBackOneByteInputStream) raw;
 				long numNewlines = StreamCopy.skipChars(pb1bis, CR_NL_CHARS);
 				if(numNewlines > 0) {
-					metaData.putLong(PAYLOAD_LENGTH, countingIS.getCount());
+					long payloadLength = countingIS.getCount();
+					if (!metaData.has(PAYLOAD_LENGTH) || payloadLength != metaData.getLong(PAYLOAD_LENGTH)) {
+						metaData.putLong(PAYLOAD_LENGTH, payloadLength);
+					}
 					metaData.putLong(PAYLOAD_SLOP_BYTES, numNewlines);
 					metaData.putString(PAYLOAD_DIGEST, "sha1:"+digString);
 				}
 			}
 		}
 	}
+
 	public MetaData getEnvelopeMetaData() {
 		return envelope;
 	}
diff --git a/src/main/java/org/archive/resource/warc/record/WARCMetaDataResourceFactory.java b/src/main/java/org/archive/resource/warc/record/WARCMetaDataResourceFactory.java
@@ -21,6 +21,7 @@ public WARCMetaDataResourceFactory() {
 		parser = new HttpHeaderParser();
 	}
 
+	@Override
 	public Resource getResource(InputStream is, MetaData parentMetaData,
 			ResourceContainer container) throws ResourceParseException,
 			IOException {
@@ -33,8 +34,13 @@ public Resource getResource(InputStream is, MetaData parentMetaData,
 			if(headers.isCorrupt()) {
 				md.putBoolean(WARC_META_FIELDS_CORRUPT, true);
 			}
-			parentMetaData.putLong(PAYLOAD_SLOP_BYTES, StreamCopy.readToEOF(is));		
-			parentMetaData.putLong(PAYLOAD_LENGTH, bytes);
+			long trailingSlopBytes = StreamCopy.readToEOF(is);
+			if (!parentMetaData.has(PAYLOAD_SLOP_BYTES) || trailingSlopBytes > 0) {
+				parentMetaData.putLong(PAYLOAD_SLOP_BYTES, trailingSlopBytes);
+			}
+			if (!parentMetaData.has(PAYLOAD_LENGTH) || bytes != parentMetaData.getLong(PAYLOAD_LENGTH)) {
+				parentMetaData.putLong(PAYLOAD_LENGTH, bytes);
+			}
 			return new WARCMetaDataResource(md,container, headers);
 			
 		} catch (HttpParseException e) {
diff --git a/src/test/java/org/archive/resource/arc/ARCResourceTest.java b/src/test/java/org/archive/resource/arc/ARCResourceTest.java
@@ -0,0 +1,48 @@
+package org.archive.resource.arc;
+
+
+import static org.archive.resource.ResourceConstants.PAYLOAD_LENGTH;
+import static org.archive.resource.ResourceConstants.PAYLOAD_SLOP_BYTES;
+
+import java.io.IOException;
+
+import org.archive.extract.ExtractingResourceFactoryMapper;
+import org.archive.extract.ExtractingResourceProducer;
+import org.archive.extract.ProducerUtils;
+import org.archive.extract.ResourceFactoryMapper;
+import org.archive.resource.Resource;
+import org.archive.resource.ResourceParseException;
+import org.archive.resource.ResourceProducer;
+import org.archive.util.StreamCopy;
+
+import com.github.openjson.JSONObject;
+
+import junit.framework.TestCase;
+
+public class ARCResourceTest extends TestCase {
+
+	public void testARCResource() throws ResourceParseException, IOException {
+		String testFileName = "../../format/arc/IAH-20080430204825-00000-blackbook-truncated.arc";
+		ResourceProducer producer = ProducerUtils.getProducer(getClass().getResource(testFileName).getPath());
+		ResourceFactoryMapper mapper = new ExtractingResourceFactoryMapper();
+		ExtractingResourceProducer extractor = new ExtractingResourceProducer(producer, mapper);
+
+		Resource resource = extractor.getNext();
+
+		while (resource != null) {
+			JSONObject payloadMD = resource.getMetaData().getTopMetaData().getJSONObject("Envelope")
+					.getJSONObject("Payload-Metadata");
+			System.err.println(payloadMD);
+
+			if (payloadMD.has(PAYLOAD_LENGTH)) {
+				assertTrue(payloadMD.getLong(PAYLOAD_LENGTH) != -1);
+			}
+			if (payloadMD.has(PAYLOAD_SLOP_BYTES)) {
+				// does not occur with the tested ARC file
+			}
+
+			StreamCopy.readToEOF(resource.getInputStream());
+			resource = extractor.getNext();
+		}
+	}
+}
diff --git a/src/test/java/org/archive/resource/warc/WARCResourceTest.java b/src/test/java/org/archive/resource/warc/WARCResourceTest.java
@@ -0,0 +1,46 @@
+package org.archive.resource.warc;
+
+import static org.archive.resource.ResourceConstants.PAYLOAD_LENGTH;
+import static org.archive.resource.ResourceConstants.PAYLOAD_SLOP_BYTES;
+
+import java.io.IOException;
+
+import org.archive.extract.ExtractingResourceFactoryMapper;
+import org.archive.extract.ExtractingResourceProducer;
+import org.archive.extract.ProducerUtils;
+import org.archive.extract.ResourceFactoryMapper;
+import org.archive.resource.Resource;
+import org.archive.resource.ResourceParseException;
+import org.archive.resource.ResourceProducer;
+import org.archive.util.StreamCopy;
+
+import com.github.openjson.JSONObject;
+
+import junit.framework.TestCase;
+
+public class WARCResourceTest extends TestCase {
+
+	public void testWARCResource() throws ResourceParseException, IOException {
+		String testFileName = "../../format/warc/IAH-urls-wget.warc";
+		ResourceProducer producer = ProducerUtils.getProducer(getClass().getResource(testFileName).getPath());
+		ResourceFactoryMapper mapper = new ExtractingResourceFactoryMapper();
+		ExtractingResourceProducer extractor = new ExtractingResourceProducer(producer, mapper);
+
+		Resource resource = extractor.getNext();
+
+		while (resource != null) {
+			JSONObject payloadMD = resource.getMetaData().getTopMetaData().getJSONObject("Envelope")
+					.getJSONObject("Payload-Metadata");
+
+			if (payloadMD.has(PAYLOAD_LENGTH)) {
+				assertTrue(payloadMD.getLong(PAYLOAD_LENGTH) != -1);
+			}
+			if (payloadMD.has(PAYLOAD_SLOP_BYTES)) {
+				assertEquals(4, payloadMD.getLong(PAYLOAD_SLOP_BYTES));
+			}
+
+			StreamCopy.readToEOF(resource.getInputStream());
+			resource = extractor.getNext();
+		}
+	}
+}

Original file line number	Diff line number	Diff line change
`@@ -64,10 +64,12 @@ public ARCResource(MetaData metaData, ResourceContainer container,`
`64`	`64`	`}`
`65`	`65`	`}`
`66`	`66`
	`67`	`+ @Override`
`67`	`68`	`public InputStream getInputStream() {`
`68`	`69`	`return new EOFNotifyingInputStream(digIS, this);`
`69`	`70`	`}`
`70`	`71`
	`72`	`+ @Override`
`71`	`73`	`public void notifyEOF() throws IOException {`
`72`	`74`	`metaData.putLong(PAYLOAD_LENGTH, countingIS.getCount());`
`73`	`75`	`String digString = Base32.encode(digIS.getMessageDigest().digest());`
Original file line number	Diff line number	Diff line change
`@@ -53,7 +53,7 @@ public WARCResource(MetaData metaData, ResourceContainer container,`
`53`	`53`	`countingIS = new CountingInputStream(`
`54`	`54`	`ByteStreams.limit(response, length));`
`55`	`55`	`} else {`
`56`		`- throw new ResourceParseException(null);`
	`56`	`+ throw new ResourceParseException(new Exception("Zero or negative length: " + length));`
`57`	`57`	`}`
`58`	`58`	`try {`
`59`	`59`	`digIS = new DigestInputStream(countingIS,`
`@@ -63,14 +63,18 @@ public WARCResource(MetaData metaData, ResourceContainer container,`
`63`	`63`	`}`
`64`	`64`	`}`
`65`	`65`
	`66`	`+ @Override`
`66`	`67`	`public InputStream getInputStream() {`
`67`	`68`	`return new EOFNotifyingInputStream(digIS, this);`
`68`	`69`	`}`
`69`	`70`
	`71`	`+ @Override`
`70`	`72`	`public void notifyEOF() throws IOException {`
`71`	`73`	`String digString = Base32.encode(digIS.getMessageDigest().digest());`
`72`	`74`	`if(container.isCompressed()) {`
`73`		`- metaData.putLong(PAYLOAD_LENGTH, countingIS.getCount());`
	`75`	`+ if (!metaData.has(PAYLOAD_LENGTH) \|\| countingIS.getCount() != metaData.getLong(PAYLOAD_LENGTH)) {`
	`76`	`+ metaData.putLong(PAYLOAD_LENGTH, countingIS.getCount());`
	`77`	`+ }`
`74`	`78`	`metaData.putLong(PAYLOAD_SLOP_BYTES, StreamCopy.readToEOF(response));`
`75`	`79`	`metaData.putString(PAYLOAD_DIGEST, "sha1:"+digString);`
`76`	`80`	`} else {`
`@@ -81,13 +85,17 @@ public void notifyEOF() throws IOException {`
`81`	`85`	`(PushBackOneByteInputStream) raw;`
`82`	`86`	`long numNewlines = StreamCopy.skipChars(pb1bis, CR_NL_CHARS);`
`83`	`87`	`if(numNewlines > 0) {`
`84`		`- metaData.putLong(PAYLOAD_LENGTH, countingIS.getCount());`
	`88`	`+ long payloadLength = countingIS.getCount();`
	`89`	`+ if (!metaData.has(PAYLOAD_LENGTH) \|\| payloadLength != metaData.getLong(PAYLOAD_LENGTH)) {`
	`90`	`+ metaData.putLong(PAYLOAD_LENGTH, payloadLength);`
	`91`	`+ }`
`85`	`92`	`metaData.putLong(PAYLOAD_SLOP_BYTES, numNewlines);`
`86`	`93`	`metaData.putString(PAYLOAD_DIGEST, "sha1:"+digString);`
`87`	`94`	`}`
`88`	`95`	`}`
`89`	`96`	`}`
`90`	`97`	`}`
	`98`	`+`
`91`	`99`	`public MetaData getEnvelopeMetaData() {`
`92`	`100`	`return envelope;`
`93`	`101`	`}`