diff --git a/common/core/src/test/java/zingg/common/core/executor/ExecutorTester.java b/common/core/src/test/java/zingg/common/core/executor/ExecutorTester.java index 2bb8ebfde..1e1704ddf 100644 --- a/common/core/src/test/java/zingg/common/core/executor/ExecutorTester.java +++ b/common/core/src/test/java/zingg/common/core/executor/ExecutorTester.java @@ -1,18 +1,13 @@ package zingg.common.core.executor; import java.io.IOException; -import java.rmi.NoSuchObjectException; -import java.util.Objects; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import zingg.common.client.arguments.ArgumentServiceImpl; import zingg.common.client.arguments.model.IArguments; -import zingg.common.client.arguments.model.Arguments; import zingg.common.client.ClientOptions; import zingg.common.client.ZinggClientException; -import zingg.common.client.pipe.Pipe; import zingg.common.client.util.DFObjectUtil; import zingg.common.core.executor.validate.ExecutorValidator; @@ -23,39 +18,44 @@ public class ExecutorTester{ public ZinggBase executor; public ExecutorValidator validator; protected IArguments args; - protected String configFile; protected String modelId; protected DFObjectUtil dfObjectUtil; - public ExecutorTester(ZinggBase executor,ExecutorValidator validator, String configFile, String modelId, DFObjectUtil dfObjectUtil) throws ZinggClientException, IOException { + /** + * Args are built by the test (see TestArgumentsBuilder) and handed over ready to use; + * no config file is read. + */ + public ExecutorTester(ZinggBase executor,ExecutorValidator validator, IArguments args, String modelId, DFObjectUtil dfObjectUtil) throws ZinggClientException, IOException { this.executor = executor; this.validator = validator; - this.configFile = configFile; + this.args = args; this.modelId = modelId; this.dfObjectUtil = dfObjectUtil; - setupArgs(); } - public IArguments setupArgs(String configFile, String phase) throws ZinggClientException, NoSuchObjectException { - args = new ArgumentServiceImpl(Arguments.class).loadArguments(Objects.requireNonNull(getClass().getClassLoader().getResource(configFile)).getFile()); - args = updateLocation(args); - args.setModelId(modelId); - return args; - } - - public IArguments updateLocation(IArguments args){ - for (Pipe p: args.getData()) { - if (p.getProps().containsKey("path")) { - String testOneFile = getClass().getClassLoader().getResource(p.get("path")).getFile(); - // correct the location of test data - p.setProp("path", testOneFile); - } + /** + * Gets args ready for the run: stamps the model id, then lets the tester adjust them + * through updateArgs(). + * + * The driver calls this before every execution (see TestExecutorsGeneric), so it is + * deliberately not called from the constructor - a constructor call would run before + * subclass fields are assigned. + */ + public void setupArgs() throws ZinggClientException, IOException{ + // every run gets its own model id + this.args.setModelId(modelId); + try { + updateArgs(this.args); + } catch (Exception e) { + throw new ZinggClientException("Error while preparing args for the test run: ", e); } - return args; } - public void setupArgs() throws ZinggClientException, IOException{ - this.args = setupArgs(configFile, ""); + /** + * Hook for testers that need to adjust args before the run, e.g. attach an in-memory + * dataset to a pipe. Does nothing by default. + */ + protected void updateArgs(IArguments args) throws Exception { } public void initAndExecute(S session) throws ZinggClientException { diff --git a/common/core/src/test/java/zingg/common/core/executor/FindAndLabellerExecutorTester.java b/common/core/src/test/java/zingg/common/core/executor/FindAndLabellerExecutorTester.java index f6ff840e8..edfb2305e 100644 --- a/common/core/src/test/java/zingg/common/core/executor/FindAndLabellerExecutorTester.java +++ b/common/core/src/test/java/zingg/common/core/executor/FindAndLabellerExecutorTester.java @@ -3,6 +3,7 @@ import zingg.common.client.ClientOptions; import zingg.common.client.ZFrame; import zingg.common.client.ZinggClientException; +import zingg.common.client.arguments.model.IArguments; import zingg.common.client.util.DFObjectUtil; import zingg.common.core.executor.validate.ExecutorValidator; @@ -10,8 +11,8 @@ public class FindAndLabellerExecutorTester extends MatchThresholdBasedExecutorTester { - public FindAndLabellerExecutorTester(ZinggBase ftdLabelerExecutor, ExecutorValidator ftdLabelerValidator, String configFile, String modelId, DFObjectUtil dfObjectUtil) throws ZinggClientException, IOException{ - super(ftdLabelerExecutor, ftdLabelerValidator, configFile, modelId, dfObjectUtil); + public FindAndLabellerExecutorTester(ZinggBase ftdLabelerExecutor, ExecutorValidator ftdLabelerValidator, IArguments args, String modelId, DFObjectUtil dfObjectUtil) throws ZinggClientException, IOException{ + super(ftdLabelerExecutor, ftdLabelerValidator, args, modelId, dfObjectUtil); } //need to execute until we get diff --git a/common/core/src/test/java/zingg/common/core/executor/FtdAndLabelCombinedExecutorTester.java b/common/core/src/test/java/zingg/common/core/executor/FtdAndLabelCombinedExecutorTester.java index eff87b74b..22c936bdf 100644 --- a/common/core/src/test/java/zingg/common/core/executor/FtdAndLabelCombinedExecutorTester.java +++ b/common/core/src/test/java/zingg/common/core/executor/FtdAndLabelCombinedExecutorTester.java @@ -3,6 +3,7 @@ import zingg.common.client.ClientOptions; import zingg.common.client.ZFrame; import zingg.common.client.ZinggClientException; +import zingg.common.client.arguments.model.IArguments; import zingg.common.client.util.DFObjectUtil; import zingg.common.core.executor.validate.ExecutorValidator; @@ -15,9 +16,9 @@ public class FtdAndLabelCombinedExecutorTester extends MatchThres //setting labeller properties here //ftd properties are already set by super - public FtdAndLabelCombinedExecutorTester(ZinggBase ftdExecutor, ExecutorValidator ftdValidator, String configFile, + public FtdAndLabelCombinedExecutorTester(ZinggBase ftdExecutor, ExecutorValidator ftdValidator, IArguments args, ZinggBase labelExecutor, ExecutorValidator labelValidator, String modelId, DFObjectUtil dfObjectUtil) throws ZinggClientException, IOException, NoSuchMethodException { - super(ftdExecutor, ftdValidator,configFile,modelId,dfObjectUtil); + super(ftdExecutor, ftdValidator,args,modelId,dfObjectUtil); this.labelExecutor = labelExecutor; this.labelValidator = labelValidator; } diff --git a/common/core/src/test/java/zingg/common/core/executor/MatchThresholdBasedExecutorTester.java b/common/core/src/test/java/zingg/common/core/executor/MatchThresholdBasedExecutorTester.java index 365733679..c4a50263d 100644 --- a/common/core/src/test/java/zingg/common/core/executor/MatchThresholdBasedExecutorTester.java +++ b/common/core/src/test/java/zingg/common/core/executor/MatchThresholdBasedExecutorTester.java @@ -4,6 +4,7 @@ import org.apache.commons.logging.LogFactory; import zingg.common.client.ZFrame; import zingg.common.client.ZinggClientException; +import zingg.common.client.arguments.model.IArguments; import zingg.common.client.util.DFObjectUtil; import zingg.common.core.ZinggException; import zingg.common.core.executor.validate.ExecutorValidator; @@ -21,9 +22,9 @@ public abstract class MatchThresholdBasedExecutorTester extends E protected long matchCount = 0; protected long notAMatchCount = 0; - public MatchThresholdBasedExecutorTester(ZinggBase executor, ExecutorValidator validator, String configFile, String modelId, DFObjectUtil dfObjectUtil) + public MatchThresholdBasedExecutorTester(ZinggBase executor, ExecutorValidator validator, IArguments args, String modelId, DFObjectUtil dfObjectUtil) throws ZinggClientException, IOException { - super(executor, validator, configFile, modelId, dfObjectUtil); + super(executor, validator, args, modelId, dfObjectUtil); } protected void runUntilThreshold() { diff --git a/common/core/src/test/java/zingg/common/core/executor/TestExecutorsCompound.java b/common/core/src/test/java/zingg/common/core/executor/TestExecutorsCompound.java index 94bba4994..e745cb933 100644 --- a/common/core/src/test/java/zingg/common/core/executor/TestExecutorsCompound.java +++ b/common/core/src/test/java/zingg/common/core/executor/TestExecutorsCompound.java @@ -8,6 +8,7 @@ import org.junit.jupiter.api.AfterEach; import zingg.common.client.ZinggClientException; +import zingg.common.client.arguments.model.IArguments; import zingg.common.core.executor.validate.FindAndLabelValidator; import zingg.common.core.executor.validate.TrainMatchValidator; import zingg.common.core.util.IPerformCleanUpUtil; @@ -24,14 +25,15 @@ public TestExecutorsCompound() { public List> getExecutors() throws ZinggClientException, IOException, NoSuchMethodException { FindAndLabeller findAndLabel = getFindAndLabeller(); FindAndLabelValidator falValidator = new FindAndLabelValidator(findAndLabel); - ExecutorTester et = new FindAndLabellerExecutorTester<>(findAndLabel, falValidator,getConfigFile(),getModelId(),getDFObjectUtil()); + ExecutorTester et = new FindAndLabellerExecutorTester<>(findAndLabel, falValidator,getArgs(),getModelId(),getDFObjectUtil()); executorTesterList.add(et); TrainMatcher trainMatch = getTrainMatcher(); - executorTesterList.add(new ExecutorTester(trainMatch,getTrainMatchValidator(trainMatch), getConfigFile(),getModelId(),getDFObjectUtil())); + executorTesterList.add(new ExecutorTester(trainMatch,getTrainMatchValidator(trainMatch), getArgs(),getModelId(),getDFObjectUtil())); return executorTesterList; } - public abstract String getConfigFile(); + /** The args both compound phases run on; built in code, not read from a file. */ + public abstract IArguments getArgs() throws ZinggClientException; protected abstract FindAndLabeller getFindAndLabeller() throws ZinggClientException; diff --git a/common/core/src/test/java/zingg/common/core/executor/TestExecutorsSingle.java b/common/core/src/test/java/zingg/common/core/executor/TestExecutorsSingle.java index 1f1489a51..11d085ea5 100644 --- a/common/core/src/test/java/zingg/common/core/executor/TestExecutorsSingle.java +++ b/common/core/src/test/java/zingg/common/core/executor/TestExecutorsSingle.java @@ -8,6 +8,7 @@ import org.junit.jupiter.api.AfterEach; import zingg.common.client.ZinggClientException; +import zingg.common.client.arguments.model.IArguments; import zingg.common.core.executor.validate.LabellerValidator; import zingg.common.core.executor.validate.LinkerValidator; import zingg.common.core.executor.validate.MatcherValidator; @@ -38,28 +39,30 @@ public void getBaseExecutors() throws ZinggClientException, IOException, NoSuchM TrainingDataFinder tdf = getTrainingDataFinder(); Labeller labeler = getLabeller(); - executorTesterList.add(new FtdAndLabelCombinedExecutorTester(tdf, new TrainingDataFinderValidator(tdf), getConfigFile(), + executorTesterList.add(new FtdAndLabelCombinedExecutorTester(tdf, new TrainingDataFinderValidator(tdf), getArgs(), labeler, new LabellerValidator(labeler), getModelId(), getDFObjectUtil())); Trainer trainer = getTrainer(); - executorTesterList.add(new ExecutorTester(trainer,getTrainerValidator(trainer),getConfigFile(),getModelId(),getDFObjectUtil())); + executorTesterList.add(new ExecutorTester(trainer,getTrainerValidator(trainer),getArgs(),getModelId(),getDFObjectUtil())); } public void getAdditionalExecutors() throws ZinggClientException, IOException { Matcher matcher = getMatcher(); - executorTesterList.add(new ExecutorTester(matcher,new MatcherValidator(matcher),getConfigFile(),getModelId(),getDFObjectUtil())); + executorTesterList.add(new ExecutorTester(matcher,new MatcherValidator(matcher),getArgs(),getModelId(),getDFObjectUtil())); Linker linker = getLinker(); - executorTesterList.add(new ExecutorTester(linker,new LinkerValidator(linker),getLinkerConfigFile(),getModelId(),getDFObjectUtil())); + executorTesterList.add(new ExecutorTester(linker,new LinkerValidator(linker),getLinkerArgs(),getModelId(),getDFObjectUtil())); } - public abstract String getConfigFile(); + /** The args the single phase executors run on; built in code, not read from a file. */ + public abstract IArguments getArgs() throws ZinggClientException; - public abstract String getLinkerConfigFile(); + /** Linking needs its own args - two datasets and no training samples. */ + public abstract IArguments getLinkerArgs() throws ZinggClientException; protected abstract TrainingDataFinder getTrainingDataFinder() throws ZinggClientException; diff --git a/common/core/src/test/java/zingg/common/core/executor/testData/TestArgumentsBuilder.java b/common/core/src/test/java/zingg/common/core/executor/testData/TestArgumentsBuilder.java new file mode 100644 index 000000000..ef2c4ebc2 --- /dev/null +++ b/common/core/src/test/java/zingg/common/core/executor/testData/TestArgumentsBuilder.java @@ -0,0 +1,174 @@ +package zingg.common.core.executor.testData; + +import java.util.ArrayList; +import java.util.List; + +import zingg.common.client.FieldDefinition; +import zingg.common.client.IMatchType; +import zingg.common.client.MatchTypes; +import zingg.common.client.ZinggClientException; +import zingg.common.client.arguments.model.Arguments; +import zingg.common.client.arguments.model.IArguments; +import zingg.common.client.pipe.FilePipe; +import zingg.common.client.pipe.Pipe; + +/** + * Builds the args the executor integration tests run on. + * + * These used to live in configSparkIntTest.json and configSparkLinkTest.json under + * src/test/resources. Keeping them in code means the field definitions and schemas + * are checked by the compiler, and a test can tweak one value without a second copy + * of the whole config. + * + * The paths are passed in by the caller because the test data sits on the classpath - + * only the driver knows where the class loader resolved it to. + */ +public class TestArgumentsBuilder { + + public static final String DELIMITER = ","; + public static final String BAD_RECORDS_PATH = "/tmp/bad"; + public static final float LABEL_DATA_SAMPLE_SIZE = 0.5f; + public static final int NUM_PARTITIONS = 4; + + /** + * Schema of test.csv - the record id followed by the ten compared fields. + */ + public static final String DATA_SCHEMA = "id string, fname string, lname string, stNo string, add1 string, add2 string, city string, state string, areacode string, dob string, ssn string"; + + /** + * Schema of training.csv - the two labelling columns followed by the record. + * areacode comes before state here, the other way round from DATA_SCHEMA, because + * that is the order the file itself is written in. + */ + public static final String TRAINING_SCHEMA = "z_cluster string, z_ismatch integer, id string, fname string, lname string, stNo string, add1 string, add2 string, city string, areacode string, state string, dob string, ssn string"; + + /** + * Schema of test1.csv and test2.csv - like DATA_SCHEMA but with areacode before state. + */ + public static final String LINK_DATA_SCHEMA = "id string, fname string, lname string, stNo string, add1 string, add2 string, city string, areacode string, state string, dob string, ssn string"; + + protected TestArgumentsBuilder() { + } + + /** + * Args for the single and compound phases - one dataset, plus the training samples + * the labeller seeds itself from. + */ + public static IArguments buildSingleArgs(String modelId, String zinggDir, String dataPath, + String trainingPath, String outputPath, String stopWordsPath) throws ZinggClientException { + IArguments args = buildCommonArgs(modelId, zinggDir, outputPath); + args.setData(new Pipe[] { csvPipe("test", dataPath, DATA_SCHEMA) }); + + Pipe trainingPipe = csvPipe("trainingPos", trainingPath, TRAINING_SCHEMA); + trainingPipe.setProp("badRecordsPath", BAD_RECORDS_PATH); + args.setTrainingSamples(new Pipe[] { trainingPipe }); + + args.setFieldDefinition(getMatchFieldDefinition(stopWordsPath)); + return args; + } + + /** + * Args for the link phase - two datasets to link across, and no training samples, + * since linking runs off the model the earlier phases trained. + */ + public static IArguments buildLinkArgs(String modelId, String zinggDir, String data1Path, + String data2Path, String outputPath) throws ZinggClientException { + IArguments args = buildCommonArgs(modelId, zinggDir, outputPath); + args.setData(new Pipe[] { + csvPipe("test1", data1Path, LINK_DATA_SCHEMA), + csvPipe("test2", data2Path, LINK_DATA_SCHEMA) + }); + args.setFieldDefinition(getLinkFieldDefinition()); + return args; + } + + protected static IArguments buildCommonArgs(String modelId, String zinggDir, String outputPath) + throws ZinggClientException { + IArguments args = new Arguments(); + args.setModelId(modelId); + args.setZinggDir(zinggDir); + args.setNumPartitions(NUM_PARTITIONS); + args.setLabelDataSampleSize(LABEL_DATA_SAMPLE_SIZE); + + Pipe outputPipe = new Pipe(); + outputPipe.setName("output"); + outputPipe.setFormat(Pipe.FORMAT_CSV); + outputPipe.setProp(FilePipe.PATH, outputPath); + outputPipe.setProp(FilePipe.DELIMITER, DELIMITER); + outputPipe.setProp(FilePipe.HEADER, "true"); + args.setOutput(new Pipe[] { outputPipe }); + + return args; + } + + /** + * id is carried through the output but not compared; everything else is fuzzy. + * add1 gets the stop word list so the stop word removal path is exercised too. + */ + protected static List getMatchFieldDefinition(String stopWordsPath) { + List fieldDefinition = new ArrayList(); + fieldDefinition.add(dontUseField("id")); + fieldDefinition.add(fuzzyField("fname")); + fieldDefinition.add(fuzzyField("lname")); + fieldDefinition.add(fuzzyField("stNo")); + + FieldDefinition add1 = fuzzyField("add1"); + add1.setStopWords(stopWordsPath); + fieldDefinition.add(add1); + + fieldDefinition.add(fuzzyField("add2")); + fieldDefinition.add(fuzzyField("city")); + fieldDefinition.add(fuzzyField("areacode")); + fieldDefinition.add(fuzzyField("state")); + fieldDefinition.add(fuzzyField("dob")); + fieldDefinition.add(fuzzyField("ssn")); + return fieldDefinition; + } + + /** + * Linking compares the ten fields and leaves id out of the definition altogether. + */ + protected static List getLinkFieldDefinition() { + List fieldDefinition = new ArrayList(); + fieldDefinition.add(fuzzyField("fname")); + fieldDefinition.add(fuzzyField("lname")); + fieldDefinition.add(fuzzyField("stNo")); + fieldDefinition.add(fuzzyField("add1")); + fieldDefinition.add(fuzzyField("add2")); + fieldDefinition.add(fuzzyField("city")); + fieldDefinition.add(fuzzyField("areacode")); + fieldDefinition.add(fuzzyField("state")); + fieldDefinition.add(fuzzyField("dob")); + fieldDefinition.add(fuzzyField("ssn")); + return fieldDefinition; + } + + protected static Pipe csvPipe(String name, String path, String schema) { + Pipe pipe = new Pipe(); + pipe.setName(name); + pipe.setFormat(Pipe.FORMAT_CSV); + pipe.setProp(FilePipe.PATH, path); + pipe.setProp(FilePipe.DELIMITER, DELIMITER); + pipe.setProp(FilePipe.HEADER, "false"); + pipe.setSchema(schema); + return pipe; + } + + protected static FieldDefinition fuzzyField(String name) { + return field(name, MatchTypes.FUZZY); + } + + protected static FieldDefinition dontUseField(String name) { + return field(name, MatchTypes.DONT_USE); + } + + protected static FieldDefinition field(String name, IMatchType matchType) { + FieldDefinition fieldDefinition = new FieldDefinition(); + fieldDefinition.setFieldName(name); + fieldDefinition.setFields(name); + fieldDefinition.setDataType("string"); + fieldDefinition.setMatchTypeInternal(matchType); + return fieldDefinition; + } + +} diff --git a/spark/core/src/test/java/zingg/spark/core/executor/TestSparkExecutorsCompound.java b/spark/core/src/test/java/zingg/spark/core/executor/TestSparkExecutorsCompound.java index aef6f5942..a9cc02bb4 100644 --- a/spark/core/src/test/java/zingg/spark/core/executor/TestSparkExecutorsCompound.java +++ b/spark/core/src/test/java/zingg/spark/core/executor/TestSparkExecutorsCompound.java @@ -1,6 +1,7 @@ package zingg.spark.core.executor; import java.io.IOException; +import java.util.Objects; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -12,6 +13,8 @@ import org.junit.jupiter.api.extension.ExtendWith; import zingg.common.client.ZinggClientException; +import zingg.common.client.arguments.model.IArguments; +import zingg.common.core.executor.testData.TestArgumentsBuilder; import zingg.common.client.util.DFObjectUtil; import zingg.common.client.util.IWithSession; import zingg.common.client.util.WithSession; @@ -27,8 +30,13 @@ @ExtendWith(TestSparkBaseHeavy.class) public class TestSparkExecutorsCompound extends TestExecutorsCompound,Row,Column,DataType> { - protected static final String CONFIG_FILE = "zingg/spark/core/executor/compound/configSparkIntTest.json"; protected static final String TEST_DATA_FILE = "zingg/spark/core/executor/test.csv"; + /** Left classpath relative on purpose: only the data pipes were ever resolved to a + * real path, so the training samples are read relative to the working directory. */ + protected static final String TRAINING_DATA_FILE = "./zingg/spark/core/executor/training.csv"; + protected static final String STOP_WORDS = "./zingg/spark/core/executor/stopwords/add1.csv"; + protected static final String ZINGG_DIR = "/tmp/junit_integration_spark/compound"; + protected static final String OUTPUT_DIR = "/tmp/junit_integration_spark/compound/zinggOutput"; public static final Log LOG = LogFactory.getLog(TestSparkExecutorsCompound.class); @@ -42,8 +50,14 @@ public TestSparkExecutorsCompound(SparkSession sparkSession) throws IOException, } @Override - public String getConfigFile() { - return CONFIG_FILE; + public IArguments getArgs() throws ZinggClientException { + return TestArgumentsBuilder.buildSingleArgs(getModelId(), ZINGG_DIR, resource(TEST_DATA_FILE), + TRAINING_DATA_FILE, OUTPUT_DIR, STOP_WORDS); + } + + /** test data lives on the classpath; the executors need a real path to read it from */ + protected String resource(String classpathLocation) { + return Objects.requireNonNull(getClass().getClassLoader().getResource(classpathLocation)).getFile(); } diff --git a/spark/core/src/test/java/zingg/spark/core/executor/TestSparkExecutorsSingle.java b/spark/core/src/test/java/zingg/spark/core/executor/TestSparkExecutorsSingle.java index bc245e31c..dab8419f6 100644 --- a/spark/core/src/test/java/zingg/spark/core/executor/TestSparkExecutorsSingle.java +++ b/spark/core/src/test/java/zingg/spark/core/executor/TestSparkExecutorsSingle.java @@ -1,6 +1,7 @@ package zingg.spark.core.executor; import java.io.IOException; +import java.util.Objects; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -12,6 +13,8 @@ import org.junit.jupiter.api.extension.ExtendWith; import zingg.common.client.ZinggClientException; +import zingg.common.client.arguments.model.IArguments; +import zingg.common.core.executor.testData.TestArgumentsBuilder; import zingg.common.client.util.DFObjectUtil; import zingg.common.client.util.IWithSession; import zingg.common.client.util.WithSession; @@ -28,10 +31,15 @@ @ExtendWith(TestSparkBaseHeavy.class) public class TestSparkExecutorsSingle extends TestExecutorsSingle,Row,Column,DataType> { - protected static final String CONFIG_FILE = "zingg/spark/core/executor/single/configSparkIntTest.json"; - protected static final String CONFIGLINK_FILE = "zingg/spark/core/executor/single/configSparkLinkTest.json"; + protected static final String TEST_DATA_FILE = "zingg/spark/core/executor/test.csv"; protected static final String TEST1_DATA_FILE = "zingg/spark/core/executor/test1.csv"; protected static final String TEST2_DATA_FILE = "zingg/spark/core/executor/test2.csv"; + /** Left classpath relative on purpose: only the data pipes were ever resolved to a + * real path, so the training samples are read relative to the working directory. */ + protected static final String TRAINING_DATA_FILE = "./zingg/spark/core/executor/training.csv"; + protected static final String STOP_WORDS = "./zingg/spark/core/executor/stopwords/add1.csv"; + protected static final String ZINGG_DIR = "/tmp/junit_integration_spark/single"; + protected static final String OUTPUT_DIR = "/tmp/junit_integration_spark/single/zinggOutput"; public static final Log LOG = LogFactory.getLog(TestSparkExecutorsSingle.class); private final SparkSession sparkSession; @@ -45,13 +53,20 @@ public TestSparkExecutorsSingle(SparkSession sparkSession) throws IOException, Z } @Override - public String getConfigFile() { - return CONFIG_FILE; + public IArguments getArgs() throws ZinggClientException { + return TestArgumentsBuilder.buildSingleArgs(getModelId(), ZINGG_DIR, resource(TEST_DATA_FILE), + TRAINING_DATA_FILE, OUTPUT_DIR, STOP_WORDS); } @Override - public String getLinkerConfigFile(){ - return CONFIGLINK_FILE; + public IArguments getLinkerArgs() throws ZinggClientException { + return TestArgumentsBuilder.buildLinkArgs(getModelId(), ZINGG_DIR, resource(TEST1_DATA_FILE), + resource(TEST2_DATA_FILE), OUTPUT_DIR); + } + + /** test data lives on the classpath; the executors need a real path to read it from */ + protected String resource(String classpathLocation) { + return Objects.requireNonNull(getClass().getClassLoader().getResource(classpathLocation)).getFile(); } @Override diff --git a/spark/core/src/test/resources/zingg/spark/core/executor/compound/configSparkIntTest.json b/spark/core/src/test/resources/zingg/spark/core/executor/compound/configSparkIntTest.json deleted file mode 100644 index 649df8aad..000000000 --- a/spark/core/src/test/resources/zingg/spark/core/executor/compound/configSparkIntTest.json +++ /dev/null @@ -1,107 +0,0 @@ -{ - "trainingSamples" : [{ - "name":"trainingPos", - "format":"csv", - "props": { - "path": "./zingg/spark/core/executor/training.csv", - "delimiter": ",", - "header":false, - "badRecordsPath":"/tmp/bad" - }, - "schema": "z_cluster string, z_ismatch integer, id string, fname string, lname string, stNo string, add1 string, add2 string, city string, areacode string, state string, dob string, ssn string" - }], - "fieldDefinition":[ - { - "fieldName" : "id", - "matchType" : "dont_use", - "fields" : "id", - "dataType": "string" - }, - { - "fieldName" : "fname", - "matchType" : "fuzzy", - "fields" : "fname", - "dataType": "string" - }, - { - "fieldName" : "lname", - "matchType" : "fuzzy", - "fields" : "lname", - "dataType": "string" - }, - { - "fieldName" : "stNo", - "matchType": "fuzzy", - "fields" : "stNo", - "dataType": "string" - }, - { - "fieldName" : "add1", - "matchType": "fuzzy", - "fields" : "add1", - "dataType": "string", - "stopWords": "./zingg/spark/core/executor/stopwords/add1.csv" - }, - { - "fieldName" : "add2", - "matchType": "fuzzy", - "fields" : "add2", - "dataType": "string" - }, - { - "fieldName" : "city", - "matchType": "fuzzy", - "fields" : "city", - "dataType": "string" - }, - { - "fieldName" : "areacode", - "matchType": "fuzzy", - "fields" : "areacode", - "dataType": "string" - }, - { - "fieldName" : "state", - "matchType": "fuzzy", - "fields" : "state", - "dataType": "string" - }, - { - "fieldName" : "dob", - "matchType": "fuzzy", - "fields" : "dob", - "dataType": "string" - }, - { - "fieldName" : "ssn", - "matchType": "fuzzy", - "fields" : "ssn", - "dataType": "string" - } - ], - "output" : [{ - "name":"output", - "format":"csv", - "props": { - "path": "/tmp/junit_integration_spark/compound/zinggOutput", - "delimiter": ",", - "header":true - } - }], - "data" : [{ - "name":"test", - "format":"csv", - "props": { - "path": "./zingg/spark/core/executor/test.csv", - "delimiter": ",", - "header":false - }, - "schema": "id string, fname string, lname string, stNo string, add1 string, add2 string, city string, state string, areacode string, dob string, ssn string" - } - ], - "labelDataSampleSize" : 0.5, - "numPartitions":4, - "modelId": "junit_integration_spark", - "zinggDir": "/tmp/junit_integration_spark/compound" - -} diff --git a/spark/core/src/test/resources/zingg/spark/core/executor/single/configSparkIntTest.json b/spark/core/src/test/resources/zingg/spark/core/executor/single/configSparkIntTest.json deleted file mode 100644 index 9957f0146..000000000 --- a/spark/core/src/test/resources/zingg/spark/core/executor/single/configSparkIntTest.json +++ /dev/null @@ -1,107 +0,0 @@ -{ - "trainingSamples" : [{ - "name":"trainingPos", - "format":"csv", - "props": { - "path": "./zingg/spark/core/executor/training.csv", - "delimiter": ",", - "header":false, - "badRecordsPath":"/tmp/bad" - }, - "schema": "z_cluster string, z_ismatch integer, id string, fname string, lname string, stNo string, add1 string, add2 string, city string, areacode string, state string, dob string, ssn string" - }], - "fieldDefinition":[ - { - "fieldName" : "id", - "matchType" : "dont_use", - "fields" : "id", - "dataType": "string" - }, - { - "fieldName" : "fname", - "matchType" : "fuzzy", - "fields" : "fname", - "dataType": "string" - }, - { - "fieldName" : "lname", - "matchType" : "fuzzy", - "fields" : "lname", - "dataType": "string" - }, - { - "fieldName" : "stNo", - "matchType": "fuzzy", - "fields" : "stNo", - "dataType": "string" - }, - { - "fieldName" : "add1", - "matchType": "fuzzy", - "fields" : "add1", - "dataType": "string", - "stopWords": "./zingg/spark/core/executor/stopwords/add1.csv" - }, - { - "fieldName" : "add2", - "matchType": "fuzzy", - "fields" : "add2", - "dataType": "string" - }, - { - "fieldName" : "city", - "matchType": "fuzzy", - "fields" : "city", - "dataType": "string" - }, - { - "fieldName" : "areacode", - "matchType": "fuzzy", - "fields" : "areacode", - "dataType": "string" - }, - { - "fieldName" : "state", - "matchType": "fuzzy", - "fields" : "state", - "dataType": "string" - }, - { - "fieldName" : "dob", - "matchType": "fuzzy", - "fields" : "dob", - "dataType": "string" - }, - { - "fieldName" : "ssn", - "matchType": "fuzzy", - "fields" : "ssn", - "dataType": "string" - } - ], - "output" : [{ - "name":"output", - "format":"csv", - "props": { - "path": "/tmp/junit_integration_spark/single/zinggOutput", - "delimiter": ",", - "header":true - } - }], - "data" : [{ - "name":"test", - "format":"csv", - "props": { - "path": "./zingg/spark/core/executor/test.csv", - "delimiter": ",", - "header":false - }, - "schema": "id string, fname string, lname string, stNo string, add1 string, add2 string, city string, state string, areacode string, dob string, ssn string" - } - ], - "labelDataSampleSize" : 0.5, - "numPartitions":4, - "modelId": "junit_integration_spark", - "zinggDir": "/tmp/junit_integration_spark/single" - -} diff --git a/spark/core/src/test/resources/zingg/spark/core/executor/single/configSparkLinkTest.json b/spark/core/src/test/resources/zingg/spark/core/executor/single/configSparkLinkTest.json deleted file mode 100644 index 0e05fd0f7..000000000 --- a/spark/core/src/test/resources/zingg/spark/core/executor/single/configSparkLinkTest.json +++ /dev/null @@ -1,99 +0,0 @@ -{ - "fieldDefinition":[ - { - "fieldName" : "fname", - "matchType" : "fuzzy", - "fields" : "fname", - "dataType": "string" - }, - { - "fieldName" : "lname", - "matchType" : "fuzzy", - "fields" : "lname", - "dataType": "string" - }, - { - "fieldName" : "stNo", - "matchType": "fuzzy", - "fields" : "stNo", - "dataType": "string" - }, - { - "fieldName" : "add1", - "matchType": "fuzzy", - "fields" : "add1", - "dataType": "string" - }, - { - "fieldName" : "add2", - "matchType": "fuzzy", - "fields" : "add2", - "dataType": "string" - }, - { - "fieldName" : "city", - "matchType": "fuzzy", - "fields" : "city", - "dataType": "string" - }, - { - "fieldName" : "areacode", - "matchType": "fuzzy", - "fields" : "areacode", - "dataType": "string" - }, - { - "fieldName" : "state", - "matchType": "fuzzy", - "fields" : "state", - "dataType": "string" - }, - { - "fieldName" : "dob", - "matchType": "fuzzy", - "fields" : "dob", - "dataType": "string" - }, - { - "fieldName" : "ssn", - "matchType": "fuzzy", - "fields" : "ssn", - "dataType": "string" - } - ], - "output" : [{ - "name":"output", - "format":"csv", - "props": { - "path": "/tmp/junit_integration_spark/single/zinggOutput", - "delimiter": ",", - "header":true - } - }], - "data" : [{ - "name":"test1", - "format":"csv", - "props": { - "path": "./zingg/spark/core/executor/test1.csv", - "delimiter": ",", - "header":false - }, - "schema": "id string, fname string, lname string, stNo string, add1 string, add2 string, city string, areacode string, state string, dob string, ssn string" - }, - { - "name":"test2", - "format":"csv", - "props": { - "path": "./zingg/spark/core/executor/test2.csv", - "delimiter": ",", - "header":false - }, - "schema": "id string, fname string, lname string, stNo string, add1 string, add2 string, city string, areacode string, state string, dob string, ssn string" - } - ], - "labelDataSampleSize" : 0.5, - "numPartitions":4, - "modelId": "junit_integration_spark", - "zinggDir": "/tmp/junit_integration_spark/single" - -}