Add fix variations of mf runner examples #598

Reuse all fix workflows from #654 and bring folders together as suggested by @blackwinter in #654 (comment)

Author: TobiasNx

metafacture-runner/src/main/dist/examples/beacon/create/create_fix.flux

@@ -0,0 +1,36 @@
+//creates a beacon file based on a pica+ dump of the DNB CBS data.
+
+default type = "ALL";
+default out = dump + "-" + type + ".beacon";
+default header = FLUX_DIR + "header.txt";
+
+
+//read header
+"reading header " + header | write("stdout");
+header|open-file|as-lines|@Y;
+
+//count references
+"counting references in " + dump | write("stdout");
+
+dump
+| open-file
+| as-lines
+| catch-object-exception
+| decode-pica
+| batch-log(batchsize="100000")
+| fix(FLUX_DIR + "extract.fix", *)
+| stream-to-triples(redirect="true")
+| sort-triples(by="subject")
+| collect-triples
+| fix(FLUX_DIR + "output.fix")
+| batch-log("merged ${totalRecords}", batchsize="100000")
+| stream-to-triples
+| template("${s}")
+| @Y;
+
+@Y
+| wait-for-inputs("2")
+| write(out);
+
+
+

metafacture-runner/src/main/dist/examples/beacon/create/create.flux renamed to metafacture-runner/src/main/dist/examples/beacon/create/create_morph.flux

@@ -0,0 +1,50 @@
+# 002@ not repeatable
+
+if any_match("[email protected]", "^Tp.*$")
+    copy_field("[email protected]","ok")
+end
+
+#                <!-- DBSM: -->
+#                <!-- (006U $0 “04p01*”) or (017A $a “yy”) -->
+if any_match("006U.0","04p01.*")
+        add_field("@value","DBSM|ALL")
+elsif any_equal("017A.a","yy")
+        add_field("@value","DBSM|ALL")
+
+#                 <!-- DEA: -->
+#                 <!-- (001@ $a 2””) or (209A $f “Exilarchiv” or 209A $f “HB/EB”) -->
+elsif any_equal("[email protected]","2")
+        add_field("@value","DEA|ALL")
+elsif any_equal("209A.f","HB/EB")
+        add_field("@value","DEA|ALL")
+
+#                 <!-- DMA: -->
+#                 <!-- (002@ $0 “G*” or 002@ $0 “M*”) or (006U $0 “10,P01*”) -->
+
+elsif any_equal("[email protected]","^[GM].*")
+        add_field("@value","DMA|ALL")
+
+elsif any_equal("006U.0","^10,P01.*")
+        add_field("@value","DMA|ALL")
+else
+    add_field("@value","ALL")
+end
+
+# Test if type variable fits
+
+if any_contain("@value","$[type]")
+    add_field("@value","$[type]")
+else
+    remove_field("@value")
+end
+
+do list(path: "041A*|028A*|029B*|028C*|028Q*|028P*|028F*|028M*|028D*|028E*", "var":"$i")
+    trim("$i.9")
+    to_var("$i.9","ref")
+    if exists("$i.9")
+        copy_field("@value","{to:$[ref]}refed")
+    end
+end
+
+retain("{to*","ok")
+

metafacture-runner/src/main/dist/examples/beacon/create/extract.fix

@@ -0,0 +1,8 @@
+unless exists("refed")
+    remove_field("ok","")
+end
+
+unless exists("ok")
+  remove_field("ok","")
+end
+

metafacture-runner/src/main/dist/examples/beacon/create/output.fix

@@ -0,0 +1,11 @@
+default fileName = FLUX_DIR + "gnd-sample.pica";
+
+fileName
+| open-file
+| as-lines
+| decode-pica
+| fix(FLUX_DIR + "gnd-type.fix")
+| stream-to-triples
+| count-triples(countBy="object")
+| template("${s}\t${o}")
+| write("stdout");

metafacture-runner/src/main/dist/examples/count/gnd/count-gnd-types_fix.flux

@@ -0,0 +1,6 @@
+if any_match("[email protected]","...*")
+  replace_all("[email protected]","^(..).*","$1") #only keep the first two letters
+  retain("[email protected]") # only keep the relevent element
+else
+  reject()
+end

metafacture-runner/src/main/dist/examples/count/gnd/count-gnd-types.flux renamed to metafacture-runner/src/main/dist/examples/count/gnd/count-gnd-types_morph.flux

@@ -0,0 +1,8 @@
+do list(path:"041A*|041A","var":"$i") # Until https://github.com/metafacture/metafacture-core/issues/651 is fixed one hass to add "041A"
+  copy_field("$i.9","relevantField.$append")
+end
+
+trim("relevantField.*")
+uniq("relevantField")
+
+retain("relevantField")

metafacture-runner/src/main/dist/examples/count/gnd/gnd-type.fix

@@ -0,0 +1,18 @@
+
+default counts=FLUX_DIR + "counts.dat";
+default catalogue = FLUX_DIR + "10.pica";
+
+//count references
+"counting references in " + catalogue | write("stdout");
+
+catalogue
+| open-file
+| as-lines
+| catch-object-exception
+| decode-pica
+| fix(FLUX_DIR + "references.fix")
+| stream-to-triples
+| count-triples(countBy="object")
+| write(counts);
+
+

metafacture-runner/src/main/dist/examples/count/subjects/references.fix

@@ -0,0 +1,9 @@
+if any_match("[email protected]","^Tp.*")
+  if any_match("041R.a",".*[Aa][Rr][Zz][Tt].*")
+    nothing()
+  else
+    reject()
+  end
+else
+  reject()
+end

metafacture-runner/src/main/dist/examples/count/subjects/references_fix.flux

@@ -0,0 +1,11 @@
+// opens file 'fileName', interprets the content as pica and filters the results
+
+default fileName = FLUX_DIR + "gnd-sample.pica";
+
+fileName
+| open-file
+| as-lines
+| decode-pica
+| fix(FLUX_DIR + "filter.fix") // Fix does not use the filter function but has its own filter mechanism within fix.
+| encode-formeta(style="verbose")
+| write("stdout");

metafacture-runner/src/main/dist/examples/count/subjects/references.flux renamed to metafacture-runner/src/main/dist/examples/count/subjects/references_morph.flux

@@ -0,0 +1,20 @@
+default base = "";
+default dump = FLUX_DIR + "10.pica";
+default out = base + "Ts1-Tg1-without-crisscross.txt";
+
+"counting references in " + dump | write("stdout");
+
+dump
+| open-file
+| as-lines
+| catch-object-exception
+| decode-pica
+| batch-log(batchsize="100000")
+| fix(FLUX_DIR + "extract.fix")
+| stream-to-triples(redirect="true")
+| sort-triples(by="subject")
+| collect-triples
+| fix(FLUX_DIR + "output.fix")
+| batch-log(batchsize="100000")
+| encode-csv(noquotes="true",separator=";")
+| write(out);