WAT extractor: do not extract page title from embedded SVG images

sebastian-nagel · sebastian-nagel · commit e36c876f67fb · 2024-10-14T15:05:31.000+02:00
- do not use <title> elements embedded in <svg> as page/document title - use the first non-empty <title> element to set the page/document title. This is required for documents where the <title> is not enclosed in the <head> element. Note: HTML5 allows the <head> element to be ommitted, see https://www.w3.org/TR/2011/WD-html5-20110525/syntax.html#optional-tags - overwrite the page/document title by the content of a <title> element inside the <head> element - for text extraction: define the title element as block element
diff --git a/src/main/java/org/archive/resource/html/ExtractingParseObserver.java b/src/main/java/org/archive/resource/html/ExtractingParseObserver.java
@@ -24,8 +24,10 @@ public class ExtractingParseObserver implements ParseObserver {
 	Stack<StringBuilder> openAnchorTexts;
 	StringBuilder textExtract;
 	String title = null;
+	boolean inHead = false;
 	boolean inTitle = false;
 	boolean inPre = false;
+	boolean inSVG = false;
 
 	protected static String cssUrlPatString = 
 		"url\\s*\\(\\s*([^)\\s]{1,8000}?)\\s*\\)";
@@ -59,7 +61,7 @@ public class ExtractingParseObserver implements ParseObserver {
 			"button", "canvas", "caption", "col", "colgroup", "dd", "div", "dl", "dt", "embed", "fieldset",
 			"figcaption", "figure", "footer", "form", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hgroup", "hr",
 			"li", "map", "noscript", "object", "ol", "output", "p", "pre", "progress", "section", "table", "tbody",
-			"textarea", "tfoot", "th", "thead", "tr", "ul", "video" };
+			"textarea", "tfoot", "th", "thead", "title", "tr", "ul", "video" };
 	private static final Set<String> blockElements;
 	/* inline elements which content is not melted with surrounding words */
 	private final static String[] INLINE_ELEMENTS_SPACING = { "address", "cite", "details", "datalist", "iframe", "img",
@@ -144,11 +146,17 @@ public void handleTagEmpty(TagNode tag) {
 	@Override
 	public void handleTagOpen(TagNode tag) {
 		String name = tag.getTagName();
-		if(name.equals("TITLE")) {
+		if (name.equals("HEAD")) {
+			inHead = true;
+		} else if (name.equals("TITLE")) {
 			inTitle = !tag.isEmptyXmlTag();
 			return;
 		} else if (name.equals("PRE")) {
 			inPre = true;
+		} else if (name.equals("SVG")) {
+			inSVG = true;
+		} else if (name.equals("BODY")) {
+			inHead = false;
 		}
 
 		if (blockElements.contains(name)) {
@@ -183,9 +191,11 @@ public void handleTagOpen(TagNode tag) {
 	public void handleTagClose(TagNode tag) {
 		String name = tag.getTagName();
 
-		if(inTitle) {
+		if (inTitle) {
 			inTitle = false;
-			data.setTitle(title);
+			if (!inSVG && (inHead || !data.hasTitle())) {
+				data.setTitle(title);
+			}
 			title = null;
 		}
 
@@ -222,8 +232,12 @@ public void handleTagClose(TagNode tag) {
 					data.addHref(vals);
 				}
 			}
+		} else if (tag.getTagName().equals("HEAD")) {
+			inHead = false;
 		} else if (tag.getTagName().equals("PRE")) {
 			inPre = false;
+		} else if (tag.getTagName().equals("SVG")) {
+			inSVG = false;
 		}
 	}
 
diff --git a/src/main/java/org/archive/resource/html/HTMLMetaData.java b/src/main/java/org/archive/resource/html/HTMLMetaData.java
@@ -31,9 +31,15 @@ private JSONObject getHeader() {
 	public void setBaseHref(String href) {
 		putUnlessNull(getHeader(),HTML_BASE, href);
 	}
+
 	public void setTitle(String title) {
 		putUnlessNull(getHeader(),HTML_TITLE, title);
 	}
+
+	public boolean hasTitle() {
+		return header != null && header.has(HTML_TITLE);
+	}
+
 	private void putUnlessNull(JSONObject o, String k, String v) {
 		if(o != null) {
 			try {
@@ -43,6 +49,7 @@ private void putUnlessNull(JSONObject o, String k, String v) {
 			}
 		}
 	}
+
 	public String[] LtoA(List<String> l) {
 		String[] a = new String[l.size()];
 		l.toArray(a);

Original file line number	Diff line number	Diff line change
`@@ -31,9 +31,15 @@ private JSONObject getHeader() {`
`31`	`31`	`public void setBaseHref(String href) {`
`32`	`32`	`putUnlessNull(getHeader(),HTML_BASE, href);`
`33`	`33`	`}`
	`34`	`+`
`34`	`35`	`public void setTitle(String title) {`
`35`	`36`	`putUnlessNull(getHeader(),HTML_TITLE, title);`
`36`	`37`	`}`
	`38`	`+`
	`39`	`+ public boolean hasTitle() {`
	`40`	`+ return header != null && header.has(HTML_TITLE);`
	`41`	`+ }`
	`42`	`+`
`37`	`43`	`private void putUnlessNull(JSONObject o, String k, String v) {`
`38`	`44`	`if(o != null) {`
`39`	`45`	`try {`
`@@ -43,6 +49,7 @@ private void putUnlessNull(JSONObject o, String k, String v) {`
`43`	`49`	`}`
`44`	`50`	`}`
`45`	`51`	`}`
	`52`	`+`
`46`	`53`	`public String[] LtoA(List<String> l) {`
`47`	`54`	`String[] a = new String[l.size()];`
`48`	`55`	`l.toArray(a);`