Merge branch 'develop'

IQSS · Aug 15, 2017 · 89c0a2d · 89c0a2d
2 parents f5a298b + 63f3876
commit 89c0a2d
Show file tree

Hide file tree

Showing 19 changed files with 276 additions and 231 deletions.
diff --git a/DataTagsLib/WORK/policy-models/dg-large-modular/questionnaire.dg b/DataTagsLib/WORK/policy-models/dg-large-modular/questionnaire.dg
@@ -1,67 +1,68 @@
 <*
-	DISCLAIMER:
-	NOT TO BE USED AS A LEGAL ADVICE
+DISCLAIMER:
+NOT TO BE USED AS A LEGAL ADVICE
 *>
 [#import ppra:    ppra.dg    ]
 [#import medical: medical.dg ]
 [#import ferpa:   ferpa.dg   ]
 [#import dppa:    dppa.dg    ]
 [#import gov:     government-records.dg ]
 [>global-start< ask:
-	{text: Do the data concern living persons?}
+  {text: Do the data concern living persons?}
   {answers:
     {yes:
       [set: Code=green] <-- Explicit settings of colors is going away soon!
-			[>filter-medical< ask:
-				{text: Do the data contain health or medical information?}
-			  {answers:
-				 {yes: [call: medical>start]}}
-			 ]
-			 [>filter-ppra< ask:
-			  {text: Was the data collected from students in an elementary or secondary school?}
-			  {answers:
-			    {yes: [call: ppra>start]}
-			  }
-			]
-			[>filter-ferpa< ask:
-			  {text: Do the data contain any information derived from ministry of education data?}
-			  {answers:
-			    {yes: [call: ferpa>start]}
-			}]
-			[call: gov>Gov1]
-			[call: dppa>DPPA1]
-      [>futures< todo: Future additions go here.]}
+      [>filter-medical< ask:
+        {text: Do the data contain health or medical information?}
+        {answers:
+          {yes: [call: medical>start]}}
+        ]
+        [>filter-ppra< ask:
+          {text: Was the data collected from students in an elementary or secondary school?}
+          {answers:
+            {yes: [call: ppra>start]}
+          }
+        ]
+        [>filter-ferpa< ask:
+          {text: Do the data contain any information derived from ministry of education data?}
+          {answers:
+            {yes: [call: ferpa>start]}
+          }]
+        [call: gov>Gov1]
+        [call: dppa>DPPA1]
+        [>futures< todo: Future additions go here.]}
       {no:  [set: Code=blue; Identity=noPersonData] }
-}]
-[call: dua]
-[end]
+    }]
+  [call: dua]
+  [end]
 
-[>dua< section:
-  {title: Data Use Agreements}
-	[>dua-ext< todo: Data use agreements]
-	[>duaTimeLimit< ask:
-		{text: Is there any reason why we cannot store the data indefinitely? }
-			{answers:
-				{no: [set: TimeLimit=none] }
-				{yes:
-					[ask:
-						{text:For how long should we keep the data?}
-						{answers:
-							{ indefinitely: [set: TimeLimit=none]  }
-							{ 50 years:     [set: TimeLimit=_50yr] }
-							{ 5 years:      [set: TimeLimit=_5yr]  }
-							{ 2 years:      [set: TimeLimit=_2yr]  }
-							{ 1 year:       [set: TimeLimit=_1yr]  }
-						}]}}]
-	[>duaAdditional< ask:
-		{text: Did the data have any restrictions on sharing, e.g. stated in an agreement or policy statement?}
-		{answers:
-			{yes: [set: ContractOrPolicy=yes]}
-			{no:  [set: ContractOrPolicy=no]}}
-	]
+  [>dua< section:
+    {title: Data Use Agreements}
+    [>dua-ext< todo: Data use agreements]
+    [>duaTimeLimit< ask:
+      {text: Is there any reason why we cannot store the data indefinitely? }
+      {answers:
+        {no: [set: TimeLimit=none] }
+        {yes:
+          [ask:
+            {text:For how long should we keep the data?}
+            {answers:
+              { indefinitely: [set: TimeLimit=none]  }
+              { 50 years:     [set: TimeLimit=_50yr] }
+              { 5 years:      [set: TimeLimit=_5yr]  }
+              { 2 years:      [set: TimeLimit=_2yr]  }
+              { 1 year:       [set: TimeLimit=_1yr]  }
+            }]}}]
+  [>duaAdditional< ask:
+    {text: Did the data have any restrictions on sharing, e.g. stated in an agreement or policy statement?}
+    {answers:
+      {yes: [set: ContractOrPolicy=yes]}
+      {no:  [set: ContractOrPolicy=no]}
+    }
+  ]
 ]
 [end]
 <*
-	DISCLAIMER:
-	NOT TO BE USED AS A LEGAL ADVICE
+DISCLAIMER:
+NOT TO BE USED AS A LEGAL ADVICE
 *>
diff --git a/DataTagsLib/WORK/policy-models/import/cats.dg b/DataTagsLib/WORK/policy-models/import/cats.dg
@@ -1,3 +1,9 @@
+[>iii< ask:
+  {text: Noname node #1 }
+  {answers:
+      {yes: [todo: todo in cats.dg]}
+  }
+]
 [>cat< section:
   {title: Cats}
   [>q-cats-group< ask:

diff --git a/DataTagsLib/WORK/policy-models/import/dogs.dg b/DataTagsLib/WORK/policy-models/import/dogs.dg
@@ -1,3 +1,9 @@
+[>WW< ask:
+  {text: Noname node #1 }
+  {answers:
+      {yes: [todo: first TODO in dogs.dg]}
+  }
+]
 [>dog< section:
   {title: Dogs!}
   [>q-dogType< ask:

diff --git a/DataTagsLib/WORK/policy-models/import/graph.dg b/DataTagsLib/WORK/policy-models/import/graph.dg
@@ -1,11 +1,10 @@
 [#import dogs: dogs.dg]
 [#import cats: cats.dg]
-
 [>q-order< ask:
   {text: Do the dogs first?}
   {answers:
-      {yes: [call: dogs>dog][call: cats>cat]}
-      {no:  [call: cats>cat][call:dogs>dog]}
+      {yes: [call: dogs>dog]<*[call: cats>cat]*>}
+      {no:  <*[call: cats>cat]*>[call:dogs>dog]}
   }
 ]
 [todo: specify mice]

diff --git a/DataTagsLib/nbproject/private/configs/CLI-hello.properties b/DataTagsLib/nbproject/private/configs/CLI-hello.properties
@@ -1,2 +1,2 @@
-application.args=policy-models/test
+application.args=policy-models/import
 work.dir=WORK
diff --git a/DataTagsLib/nbproject/private/private.properties b/DataTagsLib/nbproject/private/private.properties
@@ -6,5 +6,5 @@ javadoc.preview=true
 <<<<<<< HEAD
 user.properties.file=/Users/michael/Library/Application Support/NetBeans/8.1/build.properties
 =======
-user.properties.file=C:\\Users\\user\\AppData\\Roaming\\NetBeans\\8.2\\build.properties
+user.properties.file=/Users/michael/Library/Application Support/NetBeans/8.2/build.properties
 >>>>>>> 3e85a56f4c480fd33c660bb30999ba6bf4a8e64b
diff --git a/DataTagsLib/nbproject/private/private.xml b/DataTagsLib/nbproject/private/private.xml
@@ -10,15 +10,7 @@
             <file>file:/C:/Users/user/Desktop/DataTagging/DataTaggingLibrary%20-%20import/DataTaggingLibrary/DataTagsLib/src/edu/harvard/iq/datatags/parser/decisiongraph/CompilationUnit.java</file>
         </group>
         <group name="DataTags">
-            <file>file:/Users/michael/Documents/PhD/IQSS/Data-Tags/DataTaggingLibrary/DataTagsLib/src/edu/harvard/iq/datatags/model/graphs/DecisionGraph.java</file>
-            <file>file:/Users/michael/Documents/PhD/IQSS/Data-Tags/DataTaggingLibrary/DataTagsLib/src/edu/harvard/iq/datatags/model/types/AtomicSlot.java</file>
-            <file>file:/Users/michael/Documents/PhD/IQSS/Data-Tags/DataTaggingLibrary/DataTagsLib/test/edu/harvard/iq/datatags/runtime/ChartRunningTest.java</file>
-            <file>file:/Users/michael/Documents/PhD/IQSS/Data-Tags/DataTaggingLibrary/DataTagsLib/src/edu/harvard/iq/datatags/model/values/AtomicValue.java</file>
-            <file>file:/Users/michael/Documents/PhD/IQSS/Data-Tags/DataTaggingLibrary/DataTagsLib/src/edu/harvard/iq/datatags/parser/decisiongraph/CompilationUnit.java</file>
-            <file>file:/Users/michael/Documents/PhD/IQSS/Data-Tags/DataTaggingLibrary/DataTagsLib/src/edu/harvard/iq/datatags/parser/decisiongraph/DecisionGraphCompiler.java</file>
-            <file>file:/Users/michael/Documents/PhD/IQSS/Data-Tags/DataTaggingLibrary/DataTagsLib/src/edu/harvard/iq/datatags/model/values/AggregateValue.java</file>
-            <file>file:/Users/michael/Documents/PhD/IQSS/Data-Tags/DataTaggingLibrary/DataTagsLib/test/edu/harvard/iq/datatags/parser/decisiongraph/DecisionGraphParseResultTest.java</file>
-            <file>file:/Users/michael/Documents/PhD/IQSS/Data-Tags/DataTaggingLibrary/DataTagsLib/src/edu/harvard/iq/datatags/model/values/CompoundValue.java</file>
+            <file>file:/Users/michael/Documents/PhD/IQSS/Data-Tags/DataTaggingLibrary/DataTagsLib/src/edu/harvard/iq/datatags/tools/processors/EndNodeOptimizer.java</file>
         </group>
     </open-files>
 </project-private>
diff --git a/DataTagsLib/src/edu/harvard/iq/datatags/cli/commands/StatisticsCommand.java b/DataTagsLib/src/edu/harvard/iq/datatags/cli/commands/StatisticsCommand.java
@@ -6,7 +6,11 @@
 import edu.harvard.iq.datatags.model.types.CompoundSlot;
 import edu.harvard.iq.datatags.model.types.SlotType;
 import edu.harvard.iq.datatags.model.types.ToDoSlot;
+import static edu.harvard.iq.datatags.util.CollectionHelper.C;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
 
 /**
  * Prints statistics about the questionnaire.
@@ -31,6 +35,18 @@ public void execute(CliRunner rnr, List<String> args) throws Exception {
         rnr.println("Slot count: %d", cnt.slotsCount);
         rnr.println("Value count: %d", cnt.valuesCount);
         rnr.println("Decision graph nodes: %d", rnr.getModel().getDecisionGraph().nodeIds().size() );
+        final Map<Class<?>, AtomicInteger> countsByClass = new HashMap<>();
+        rnr.getModel().getDecisionGraph().nodes().forEach( 
+                nd -> countsByClass.computeIfAbsent(nd.getClass(), c -> new AtomicInteger(0)).incrementAndGet() );
+
+        final Map<String, Integer> counts = new HashMap<>();
+        countsByClass.entrySet().forEach( ent -> {
+            String className = C.last(ent.getKey().getName().split("\\."));
+            counts.put(className, ent.getValue().intValue());
+        });
+
+        counts.entrySet().stream().sorted((e1,e2)->e2.getValue().compareTo(e1.getValue()))
+                .forEach( ent -> rnr.println("  %s\t%d", ent.getKey(), ent.getValue()));
     }
 
 }

diff --git a/DataTagsLib/src/edu/harvard/iq/datatags/mains/DecisionGraphValidations.java b/DataTagsLib/src/edu/harvard/iq/datatags/mains/DecisionGraphValidations.java
@@ -13,7 +13,7 @@
 import edu.harvard.iq.datatags.parser.decisiongraph.ast.AstNode;
 import edu.harvard.iq.datatags.tools.DecisionGraphAstValidator;
 import edu.harvard.iq.datatags.tools.NodeValidationMessage;
-import edu.harvard.iq.datatags.tools.RepeatIdValidator;
+import edu.harvard.iq.datatags.tools.DuplicateIdValidator;
 import edu.harvard.iq.datatags.tools.UnreachableNodeValidator;
 import edu.harvard.iq.datatags.tools.ValidationMessage;
 import edu.harvard.iq.datatags.visualizers.graphviz.GraphvizGraphNodeAstVisualizer;
@@ -66,7 +66,7 @@ public static void main(String[] args) {
             System.out.println("===============");
 
             System.out.println("Validating repeating ids");
-            RepeatIdValidator riv = new RepeatIdValidator();
+            DuplicateIdValidator riv = new DuplicateIdValidator();
             List<ValidationMessage> repeatIdMessages = riv.validate(refs);
             if (repeatIdMessages.size() > 0) {
                 System.out.println(repeatIdMessages);

diff --git a/DataTagsLib/src/edu/harvard/iq/datatags/parser/PolicyModelLoader.java b/DataTagsLib/src/edu/harvard/iq/datatags/parser/PolicyModelLoader.java
@@ -14,7 +14,7 @@
 import edu.harvard.iq.datatags.tools.DecisionGraphAstValidator;
 import edu.harvard.iq.datatags.tools.DecisionGraphValidator;
 import edu.harvard.iq.datatags.tools.DuplicateNodeAnswerValidator;
-import edu.harvard.iq.datatags.tools.RepeatIdValidator;
+import edu.harvard.iq.datatags.tools.DuplicateIdValidator;
 import edu.harvard.iq.datatags.tools.UnreachableNodeValidator;
 import edu.harvard.iq.datatags.tools.ValidationMessage;
 import edu.harvard.iq.datatags.tools.processors.YesNoAnswersSorter;
@@ -32,7 +32,7 @@
  * Loads policy models from {@link PolicyModelData}. Each loader has a list of
  * validators and post-processors that is runs in the appropriate places.
  * 
- * Use the class' statis methods to obtain instances that fit certain contexts 
+ * Use the class' static methods to obtain instances that fit certain contexts 
  * (e.g. dev, production).
  * 
  * @author michael
@@ -50,7 +50,7 @@ public static PolicyModelLoader verboseLoader() {
         PolicyModelLoader res = new PolicyModelLoader();
 
         res.add( new DuplicateNodeAnswerValidator() );
-        res.add( new RepeatIdValidator() );
+        res.add(new DuplicateIdValidator() );
 
         res.add( new UnreachableNodeValidator() );
 
@@ -64,7 +64,7 @@ public static PolicyModelLoader productionLoader() {
         PolicyModelLoader res = new PolicyModelLoader();
 
         res.add( new DuplicateNodeAnswerValidator() );
-        res.add( new RepeatIdValidator() );
+        res.add(new DuplicateIdValidator() );
 
 
         res.add( new EndNodeOptimizer() );
@@ -124,22 +124,23 @@ public PolicyModelLoadResult load( PolicyModelData data ) {
                 }
                 model.setDecisionGraph(dg);
 
+                // Load localizations
+                Path localizations;
+                try {
+                    localizations = ciResolve(data.getMetadataFile().getParent(), LocalizationLoader.LOCALIZATION_DIRECTORY_NAME);
+                    if ( localizations != null ) {
+                        Files.list(localizations).filter(Files::isDirectory)
+                                                 .map(p->p.getFileName().toString())
+                                                 .forEach(res.getModel()::addLocalization);
+                    }
+                } catch (IOException ex) {
+                    res.addMessage( new ValidationMessage(Level.WARNING, "IO Error reading localizations: " + ex.getMessage()));
+                }
+
             } else {
                 res.addMessage( new ValidationMessage(Level.ERROR, "Failed to create decision graph; see previous errors.") );
             }
 
-            // Load localizations
-            Path localizations;
-            try {
-                localizations = ciResolve(data.getMetadataFile().getParent(), LocalizationLoader.LOCALIZATION_DIRECTORY_NAME);
-                if ( localizations != null ) {
-                    Files.list(localizations).filter(Files::isDirectory)
-                                             .map(p->p.getFileName().toString())
-                                             .forEach(res.getModel()::addLocalization);
-                }
-            } catch (IOException ex) {
-                res.addMessage( new ValidationMessage(Level.WARNING, "IO Error reading localizations: " + ex.getMessage()));
-            }
 
         } catch (NoSuchFileException ex) {
             res.addMessage( new ValidationMessage(Level.ERROR, "File " + ex.getMessage() + " cannot be found."));

diff --git a/DataTagsLib/src/edu/harvard/iq/datatags/parser/decisiongraph/DecisionGraphCompiler.java b/DataTagsLib/src/edu/harvard/iq/datatags/parser/decisiongraph/DecisionGraphCompiler.java
@@ -37,10 +37,15 @@ public class DecisionGraphCompiler {
     /** ID of the main compilation unit. */
     public static final String MAIN_CU_ID = "<main>";
 
+    /**
+     * Id of the synthetic [end] node added to all compiled graphs.
+     */
+    public static final String SYNTHETIC_END_NODE_ID = "[SYN-END]";
+
     /**
      * "Miranda node" - added to nodes that do not have their own end node.
      */
-    EndNode endAll = new EndNode("[SYN-END]");
+    EndNode endAll = new EndNode(SYNTHETIC_END_NODE_ID);
 
     /**
      * Maps a name of a slot to its fully qualified version (i.e from the top