Merge branch 'master' into hybrid_search

Author: tschaume

.github/workflows/testing.yml

@@ -66,6 +66,6 @@ jobs:
     env:
       GITHUB_TOKEN: ${{ secrets.PAT }}
     steps:
-      - uses: rymndhng/release-on-push-action@v0.25.0
+      - uses: rymndhng/release-on-push-action@v0.28.0
         with:
           bump_version_scheme: norelease

mpcontribs-api/Dockerfile

@@ -1,4 +1,4 @@
-FROM materialsproject/devops:python-3.1110.3 AS base
+FROM materialsproject/devops:python-3.1110.8 AS base
 RUN apt-get update && apt-get install -y --no-install-recommends supervisor libopenblas-dev libpq-dev vim && apt-get clean
 WORKDIR /app
 
@@ -12,11 +12,13 @@ COPY mpcontribs mpcontribs
 RUN pip install $PIP_FLAGS --no-deps .
 #ENV SETUPTOOLS_SCM_PRETEND_VERSION 0.0.0
 #COPY marshmallow-mongoengine marshmallow-mongoengine
-#RUN cd marshmallow-mongoengine && pip install $PIP_FLAGS --no-deps .
+#RUN cd marshmallow-mongoengine && pip install $PIP_FLAGS --no-deps -e .
 #COPY mimerender mimerender
-#RUN cd mimerender && pip install $PIP_FLAGS --no-deps .
+#RUN cd mimerender && pip install $PIP_FLAGS --no-deps -e .
 #COPY flask-mongorest flask-mongorest
-#RUN cd flask-mongorest && pip install $PIP_FLAGS --no-deps .
+#RUN cd flask-mongorest && pip install $PIP_FLAGS --no-deps -e .
+#COPY AtlasQ AtlasQ
+#RUN cd AtlasQ && pip install $PIP_FLAGS --no-deps -e .
 RUN wget -q https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh && \
   chmod +x wait-for-it.sh && mv wait-for-it.sh /usr/local/bin/ && \
   wget -q https://github.com/materialsproject/MPContribs/blob/master/mpcontribs-api/mpcontribs/api/contributions/formulae.json.gz?raw=true \

mpcontribs-api/mpcontribs/api/contributions/document.py

@@ -4,6 +4,7 @@
 
 from hashlib import md5
 from math import isnan
+from bson.dbref import DBRef
 from datetime import datetime
 from flask import current_app
 from atlasq import AtlasManager, AtlasQ
@@ -18,8 +19,6 @@
 from boltons.iterutils import remap
 from decimal import Decimal
 from pint import UnitRegistry
-from pint.unit import UnitDefinition
-from pint.converters import ScaleConverter
 from pint.errors import DimensionalityError
 from uncertainties import ufloat_fromstr
 from pymatgen.core import Composition, Element
@@ -35,12 +34,18 @@
         lambda s: s.replace("%", " percent "),
     ],
 )
-ureg.default_format = "~,P"
-
-ureg.define(UnitDefinition("percent", "%", (), ScaleConverter(0.01)))
-ureg.define(UnitDefinition("permille", "%%", (), ScaleConverter(0.001)))
-ureg.define(UnitDefinition("ppm", "ppm", (), ScaleConverter(1e-6)))
-ureg.define(UnitDefinition("ppb", "ppb", (), ScaleConverter(1e-9)))
+ureg.formatter.default_format = "~,P"
+
+if "percent" not in ureg:
+    # percent is native in pint >= 0.21
+    ureg.define("percent = 0.01 = %")
+if "permille" not in ureg:
+    # permille is native in pint >= 0.24.2
+    ureg.define("permille = 0.001 = ‰ = %%")
+if "ppm" not in ureg:
+    # ppm is native in pint >= 0.21
+    ureg.define("ppm = 1e-6")
+ureg.define("ppb = 1e-9")
 ureg.define("atom = 1")
 ureg.define("bohr_magneton = e * hbar / (2 * m_e) = µᵇ = µ_B = mu_B")
 ureg.define("electron_mass = 9.1093837015e-31 kg = mₑ = m_e")
@@ -68,12 +73,12 @@ def format_cell(cell):
         return cell
 
     q = get_quantity(cell)
-    if not q or isnan(q.nominal_value):
+    if not q or isnan(q.magnitude.nominal_value):
         return cell
 
     q = truncate_digits(q)
     try:
-        return str(q.nominal_value) if isnan(q.std_dev) else str(q)
+        return str(q.magnitude.nominal_value) if isnan(q.magnitude.std_dev) else str(q)
     except Exception:
         return cell
 
@@ -102,10 +107,10 @@ def get_quantity(s):
 
 
 def truncate_digits(q):
-    if isnan(q.nominal_value):
+    if isnan(q.magnitude.nominal_value):
         return q
 
-    v = Decimal(str(q.nominal_value))
+    v = Decimal(str(q.magnitude.nominal_value))
     vt = v.as_tuple()
 
     if vt.exponent >= 0:
@@ -114,8 +119,8 @@ def truncate_digits(q):
     dgts = len(vt.digits)
     dgts = max_dgts if dgts > max_dgts else dgts
     s = f"{v:.{dgts}g}"
-    if not isnan(q.std_dev):
-        s += f"+/-{q.std_dev:.{dgts}g}"
+    if not isnan(q.magnitude.std_dev):
+        s += f"+/-{q.magnitude.std_dev:.{dgts}g}"
 
     if q.units:
         s += f" {q.units}"
@@ -260,7 +265,7 @@ def make_quantities(path, key, value):
                 return key, value
 
             # silently ignore "nan"
-            if isnan(q.nominal_value):
+            if isnan(q.magnitude.nominal_value):
                 return False
 
             # ensure that the same units are used across contributions
@@ -288,11 +293,11 @@ def make_quantities(path, key, value):
             q = truncate_digits(q)
 
             # return new value dict
-            display = str(q.value) if isnan(q.std_dev) else str(q)
+            display = str(q.value) if isnan(q.magnitude.std_dev) else str(q)
             value = {
                 "display": display,
-                "value": q.nominal_value,
-                "error": q.std_dev,
+                "value": q.magnitude.nominal_value,
+                "error": q.magnitude.std_dev,
                 "unit": str(q.units),
             }
             return key, value
@@ -307,10 +312,11 @@ def pre_delete(cls, sender, document, **kwargs):
         document.reload(*args)
 
         for component in COMPONENTS.keys():
-            # check if other contributions exist before deletion!
-            for idx, obj in enumerate(getattr(document, component)):
+            # check if other contributions exist before deletion
+            # and make sure component still exists (getattr converts ref to object)
+            for obj in getattr(document, component):
                 q = {component: obj.id}
-                if sender.objects(**q).count() < 2:
+                if sender.objects(**q).count() < 2 and not isinstance(obj, DBRef):
                     obj.delete()
 
 

mpcontribs-api/mpcontribs/api/contributions/views.py

@@ -165,7 +165,7 @@ def has_add_permission(self, req, obj):
                 raise Unauthorized(f"Can't add {obj.identifier}: {msg}")
 
         query = dict(project=obj.project.id, identifier=obj.identifier)
-        if obj.project.unique_identifiers and Contributions.objects(query).count():
+        if obj.project.unique_identifiers and Contributions.objects(**query).count():
             raise Unauthorized(f"{obj.identifier} already added for {obj.project.id}")
 
         return True

mpcontribs-api/mpcontribs/api/projects/document.py

@@ -274,29 +274,29 @@ def post_save(cls, sender, document, **kwargs):
 
                 # resolve/lookup component fields
                 # NOTE also includes dynamic document fields
-                for component in COMPONENTS.keys():
-                    pipeline.append(
-                        {
-                            "$lookup": {
-                                "from": component,
-                                "localField": component,
-                                "foreignField": "_id",
-                                "as": component,
-                            }
-                        }
-                    )
+                # for component in COMPONENTS.keys():
+                #    pipeline.append(
+                #        {
+                #            "$lookup": {
+                #                "from": component,
+                #                "localField": component,
+                #                "foreignField": "_id",
+                #                "as": component,
+                #            }
+                #        }
+                #    )
 
                 # document size and attachment content size
                 project_stage = {
-                    "_id": 0,
-                    "size": {"$bsonSize": "$$ROOT"},
-                    "contents": {
-                        "$map": {  # attachment sizes
-                            "input": "$attachments",
-                            "as": "attm",
-                            "in": {"$toInt": "$$attm.content"},
-                        }
-                    },
+                    #    "_id": 0,
+                    #    "size": {"$bsonSize": "$$ROOT"},
+                    #    "contents": {
+                    #        "$map": {  # attachment sizes
+                    #            "input": "$attachments",
+                    #            "as": "attm",
+                    #            "in": {"$toInt": "$$attm.content"},
+                    #        }
+                    #    },
                 }
 
                 # number of components
@@ -321,14 +321,14 @@ def post_save(cls, sender, document, **kwargs):
                 pipeline.append({"$project": project_stage})
 
                 # forward fields and sum attachment contents
-                project_stage_2 = {k: 1 for k, v in project_stage.items()}
-                project_stage_2["contents"] = {"$sum": "$contents"}
+                project_stage_2 = {k: 1 for k in project_stage.keys()}
+                # project_stage_2["contents"] = {"$sum": "$contents"}
                 pipeline.append({"$project": project_stage_2})
 
                 # total size and total number of components
                 group_stage = {
                     "_id": None,
-                    "size": {"$sum": {"$add": ["$size", "$contents"]}},
+                    #    "size": {"$sum": {"$add": ["$size", "$contents"]}},
                 }
                 for component in COMPONENTS.keys():
                     group_stage[component] = {"$sum": f"${component}"}
@@ -357,7 +357,7 @@ def post_save(cls, sender, document, **kwargs):
                 # prep and save stats
                 stats_kwargs = {"columns": len(columns), "contributions": ncontribs}
                 if result and result[0]:
-                    stats_kwargs["size"] = result[0]["size"] / 1024 / 1024
+                    # stats_kwargs["size"] = result[0]["size"] / 1024 / 1024
                     for component in COMPONENTS.keys():
                         stats_kwargs[component] = result[0].get(component, 0)
                         if stats_kwargs[component] > 0: