merge development into t2wml-integration and resolve conflcits

Datasets/homicide/homicide_extra_edges.tsv

@@ -0,0 +1,3 @@
+id	node1	label	node2
+	Qcount_incidents	label	"count (incidents)"
+	Qcount_incidents	P31	Q47574

Datasets/homicide/homicide_properties.tsv

@@ -0,0 +1,4 @@
+id	node1	label	node2
+P100024-P31	P100024	P31	Q64830420
+P100024-data_type	P100024	data_type	Quantity
+P100024-label	P100024	label	"murder incidents"

Datasets/homicide/t2mwl/table-10a.yaml

@@ -1,15 +1,13 @@
 statementMapping:
   region:
-    - left: C
-      right: D
-      top: 5
-      bottom: 15
+    - range: C5:D15
       skip_rows:
         - =value[B:D, $row] == "" and value[A, $row] != ""
   template:
     item: =item[A, $row]
     property: P100024 # murder
     value: =value[$col, $row]
+    unit: Qnumber_incidents
     qualifier:
       - property: P585 #point in time
         value: =value[A, $row-$n] != "" and value[B:D, $row-$n] == "" -> value[A, $row-$n]

Datasets/homicide/t2mwl/table-10b.yaml

@@ -1,15 +1,13 @@
 statementMapping:
   region:
-    - left: D
-      right: D
-      top: 5
-      bottom: 24
+    - range: D5:D24
       skip_rows:
         - =value[B:D, $row] == "" and value[A, $row] != ""
   template:
     item: =item[A, $row-$n]
     property: P100024 # murder
     value: =value[$col, $row]
+    unit: Qnumber_incidents
     qualifier:
       - property: P585 #point in time
         value: =value[A, $row-$n] != "" and value[B:D, $row-$n] == "" -> value[A, $row-$n]

Datasets/homicide/t2mwl/table-10c.yaml

@@ -1,15 +1,13 @@
 statementMapping:
   region:
-    - left: E
-      right: E
-      top: 5
-      bottom: 24
+    - range: E5:E24
       skip_rows:
         - =value[A:D, $row] == ""
   template:
     item: =item[B, $row-$n]
     property: P100024 # murder
     value: =value[$col, $row]
+    unit: Qnumber_incidents
     qualifier:
       - property: P585 #point in time
         value: =value[A, $row-$n]

Datasets/homicide/t2mwl/table-10d.yaml

@@ -11,6 +11,7 @@ statementMapping:
     item: =item[A, $row]
     property: P100024 # murder
     value: =value[$col, $row]
+    unit: Qnumber_incidents
     qualifier:
       - property: P585 #point in time
         value: =value[A:B, $row-$n] == "" and value[C:D, $row-$n] != "" -> value[$col, $row-$n]

Datasets/homicide/t2mwl/table-10e.yaml

@@ -1,15 +1,13 @@
 statementMapping:
   region:
-    - left: C
-      right: D
-      top: 6
-      bottom: 16
+    - range: C6:D16
       skip_rows:
         - =value[A:B, $row] == "" and value[C:D, $row] != ""
   template:
     item: =item[A, $row]
     property: P100024 # murder
     value: =value[$col, $row]
+    unit: Qnumber_incidents
     qualifier:
       - property: P585 #point in time
         value: =value[A:B, $row-$n] == "" and value[C:D, $row-$n] != "" -> value[$col, $row-$n]

Datasets/homicide/t2mwl/table-1a.yaml

@@ -1,22 +1,28 @@
 statementMapping:
   region:
-    - left: D
-      right: F
-      top: 4
-      bottom: 9
+    - range: D4:F9
   template:
+    # item is the main subject, in this case the country;
+    # when is the cell in $col,$row, the corresponding 
+    # country is in column A, and in the same $row
     item: =item[A, $row]
     property: P100024 # murder
     value: =value[$col, $row]
-    #unit: D1002
+    unit: Qnumber_incidents
     qualifier:
-      - property: P585
+      - property: P585 # point in time
+        # when value is in $col, $row, the corresponding
+        # time is in the same $col column and in row 3
         value: =value[$col, 3]
         calendar: Q1985727
         precision: year
         time_zone: 0
-        format: "%Y"
+        format: "%Y" # the format of 4-digit years
       - property: P6001 # applies to people
+        # when the value is in $col, $row, the corresponding
+        # gender is in column C and the same $row
         value: =item[C, $row]
       - property: P123 #source
-        value: =item[B, $row]
+        # when the value is in $col, $row, the corresponding
+        # source is in column B and the same $row
+        value: =item[B, $row]

Datasets/homicide/t2mwl/table-1b.yaml

@@ -1,17 +1,13 @@
 statementMapping:
   region:
-    - left: D
-      right: F
-      top: 5
-      bottom: 13 # =value[B:, $] == ""
+    - range: D5:F13
       skip_rows:
         - =value[C, $row] == "Total"
-      skip_cells:
-        - =value[$col, $row] == "n/a"
   template:
     item: =item[A, $row]
     property: P100024 # murder
     value: =value[$col, $row]
+    unit: Qnumber_incidents
     qualifier:
       - property: P585 # point in time
         value: =value[$col, 4]

Datasets/homicide/t2mwl/table-2a.yaml

@@ -1,16 +1,17 @@
 statementMapping:
   region:
-    - left: E
-      right: E
-      top: 5
-      bottom: 22
+    - range: E5:E22
   template:
     item: =item[A, $row]
     property: P100024 # murder [a new property]
     value: =value[$col, $row]
-    unit: D1002
+    unit: Qnumber_incidents
     qualifier:
       - property: P585 # point in time
+        # we can access the previous columns
+        # using $col-1, this could have also 
+        # been written as
+        # value: =value[D, $row]
         value: =value[$col-1, $row]
         calendar: Q1985727
         precision: year

Datasets/homicide/t2mwl/table-2b.yaml

@@ -1,14 +1,11 @@
 statementMapping:
   region:
-    - left: E
-      right: E
-      top: 5
-      bottom: 22
+    - range: E5:E22
   template:
     item: =item[A, $row-$n]
     property: P100024 # murder [a new property]
     value: =value[$col, $row]
-    unit: D1002
+    unit: Qnumber_incidents
     qualifier:
       - property: P585 # point in time
         value: =value[$col-1, $row]

Datasets/homicide/t2mwl/table-3a.yaml

@@ -1,14 +1,11 @@
 statementMapping:
   region:
-    - left: C
-      right: H
-      top: 5
-      bottom: 7
+    - range: C5:H7
   template:
     item: =item[A, $row]
     property: P100024 # murder
     value: =value[$col, $row]
-    unit: D1002
+    unit: Qnumber_incidents
     qualifier:
       - property: P585 #point in time
         value: =value[$col, 4]

Datasets/homicide/t2mwl/table-3b.yaml

@@ -1,14 +1,11 @@
 statementMapping:
   region:
-    - left: C
-      right: H
-      top: 5
-      bottom: 7
+    - range: C5:H7
   template:
     item: =item[A, $row]
     property: P100024 # murder
     value: =value[$col, $row]
-    unit: D1002
+    unit: Qnumber_incidents
     qualifier:
       - property: P585 #point in time
         value: =value[$col, 3]

Datasets/homicide/t2mwl/table-4a.yaml

@@ -1,13 +1,11 @@
 statementMapping:
   region:
-    - left: C
-      right: H
-      top: 5
-      bottom: 7
+    - range: C5:H7
   template:
     item: =item[A, $row]
     property: P100024 # murder
     value: =value[$col, $row]
+    unit: Qnumber_incidents
     qualifier:
       - property: P585 #point in time
         value: =value[$col-$n, 3]

Datasets/homicide/t2mwl/table-4b.yaml

@@ -1,13 +1,13 @@
 statementMapping:
   region:
-    - left: C
-      right: H
-      top: 5
-      bottom: 7
+    - range: C5:H8
+      skip_rows:
+        - =value[B, $row] == "Totals"
   template:
     item: =item[A, $row]
     property: P100024 # murder
     value: =value[$col, $row]
+    unit: Qnumber_incidents
     qualifier:
       - property: P585 #point in time
         value: =value[$col-$n, 3]

Datasets/homicide/t2mwl/table-5a.yaml

@@ -1,15 +1,13 @@
 statementMapping:
   region:
-    - left: D
-      right: H
-      top: 5
-      bottom: 10
+    - range: D5:H10
       skip_columns:
         - =value[$col, 4] == "Source"
   template:
     item: =item[$col-1, 3]
     property: P100024 # murder
     value: =value[$col, $row]
+    unit: Qnumber_incidents
     qualifier:
       - property: P585 #point in time
         value: =value[A, $row-$n]

Datasets/homicide/t2mwl/table-5b.yaml

@@ -1,15 +1,13 @@
 statementMapping:
   region:
-    - left: C
-      right: G
-      top: 5
-      bottom: 10
+    - range: C5:G10
       skip_columns:
         - =value[$col, 4] == "Source"
   template:
     item: =item[$col-1, 3]
     property: P100024 # murder
     value: =value[$col, $row]
+    unit: Qnumber_incidents
     qualifier:
       - property: P585 #point in time
         value: =value[A, $row-$n]

Datasets/homicide/t2mwl/table-6.yaml

@@ -1,15 +1,13 @@
 statementMapping:
   region:
-    - left: C
-      right: J
-      top: 6
-      bottom: 8
+    - range: C6:J8
       skip_columns:
         - =value[$col, 5] == "Source"
   template:
     item: =item[$col-$n, 4]
     property: P100024 # murder
     value: =value[$col, $row]
+    unit: Qnumber_incidents
     qualifier:
       - property: P585 #point in time
         value: =value[A, $row]

Datasets/homicide/t2mwl/table-8.yaml

@@ -7,10 +7,11 @@ statementMapping:
       skip_rows:
         - =value[A, $row] != "" and value[B:D, $row] == ""
   template:
-    item: =value[A, $row-$n] != "" and value[B:D, $row-$n] == "" -> item[A, $row-$n]
-    # item: item[A, $row].instance_of Q6256 -> item[A, $row]
+    item: =value[B:D, $row-$n] == "" -> item[A, $row-$n]
+    #item: =instance_of(item[A, $row-$n], "Q6256") -> item[A, $row-$n]
     property: P100024 # murder
     value: =value[$col, $row]
+    unit: Qnumber_incidents
     qualifier:
       - property: P585 #point in time
         value: =value[A, $row]

Datasets/homicide/t2mwl/table-9.yaml

@@ -1,16 +1,13 @@
 statementMapping:
   region:
-    - left: D
-      right: D
-      top: 5
-      bottom: 23
+    - range: D5:D23
       skip_rows:
         - =value[A, $row] != "" and value[B:D, $row] == ""
-      # skip_rows: item[A, $row].instance_of Q48277
   template:
     item: =item[A, $row-$n]
     property: P100024 # murder
     value: =value[$col, $row]
+    unit: Qnumber_incidents
     qualifier:
       - property: P585 #point in time
         value: =value[C, $row]
@@ -20,9 +17,5 @@ statementMapping:
         format: "%Y"
       - property: P6001 # applies to people
         value: =value[A, $row-$n] != "" and value[B:D, $row-$n] == "" -> item[A, $row-$n]
-        # value: item[A, $row-$n].instance_of Q48277 -> item[A, $row-$n]
-        # Would be nice to write this as 
-        # value: item[A, $row-$n].instance_of Q48277
-        # so when the predicate is true, it keeps the item
       - property: P123 #source
         value: =item[B, $row]

HISTORY.md

@@ -1,6 +1,17 @@
 T2WML Standalone Server App History
 ===================================
 
+Changes in version 2.2.0:
+-------------------------
+* t2wml-api version 0.0.8
+* added support for uploading projects
+* added setting for warning for empty cells
+* when loading an invalid yaml, at least return the yaml content
+* bug fixes:
+  - added some missing utf-8 support
+  - sparql endpoint changes were not being propagated
+
+
 Changes in version 2.1.1:
 ------------------------
 * t2wml-api version 0.0.7

backend/app_config.py

@@ -15,8 +15,8 @@
     os.makedirs(DATADIR)
 
 BASEDIR = os.path.abspath(os.path.dirname(__file__))
-#if BASEDIR not in sys.path:
-#    sys.path.append(BASEDIR) #when running migrate, needed to not get import errors
+if BASEDIR not in sys.path:
+    sys.path.append(BASEDIR) #when running migrate, needed to not get import errors
 
 UPLOAD_FOLDER = os.path.join(DATADIR, "storage")
 CACHE_FOLDER=os.path.join(DATADIR, "cache")
@@ -39,7 +39,7 @@ class AppConfig:
 
 DEFAULT_SPARQL_ENDPOINT = 'https://dsbox02.isi.edu:8888/bigdata/namespace/wdq/sparql'
 GOOGLE_CLIENT_ID = '552769010846-tpv08vhddblg96b42nh6ltg36j41pln1.apps.googleusercontent.com'
-DATAMART_API_ENDPOINT = 'http://localhost:12543'
+DATAMART_API_ENDPOINT = 'http://localhost:14080'
 
 #############SQL STUFF