Merge pull request #58 from softwareunderground/pr_jan

[BUG] Add `location` when specifying `position` in welly and fix to pandas versions larger 1.4
Further fix bugs introduced by newer versions of xarray and welly

Author: Japhiolite

.github/workflows/docs.yml

@@ -47,7 +47,7 @@ jobs:
 
       - name: Setup Headless Display
         run: |
-          sudo apt-get install libgl1-mesa-glx
+          sudo apt-get update && sudo apt-get install libgl1-mesa-glx xvfb -y
           sudo apt-get install -y xvfb
           Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 &
           sleep 3

.github/workflows/main.yml

@@ -43,11 +43,7 @@ jobs:
 
     steps:
 
-      # Cancel any previous run of the test job; [pin v0.8.0 (2021-02-13)]
-      - name: Cancel Previous Runs
-        uses: styfle/cancel-workflow-action@3d86a7cc43670094ac248017207be0295edbc31d
-        with:
-          access_token: ${{ github.token }}
+
 
       - name: Set up Python
         uses: actions/setup-python@v2
@@ -57,7 +53,7 @@ jobs:
       - uses: actions/checkout@v2
       - name: Setup Headless Display
         run: |
-          sudo apt-get install libgl1-mesa-glx
+          sudo apt-get update && sudo apt-get install libgl1-mesa-glx xvfb -y
           sudo apt-get install -y xvfb
           Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 &
           sleep 3

.gitignore

@@ -85,3 +85,21 @@ venv.bak/
 /.idea/subsurface.iml
 /.idea/vcs.xml
 /.idea/other.xml
+
+.local_settings.json
+
+# User-specific stuff:
+.idea/workspace.xml
+.idea/tasks.xml
+.idea/dictionaries
+.idea/vcs.xml
+.idea/jsLibraryMappings.xml
+
+# Sensitive or high-churn files:
+.idea/dataSources.ids
+.idea/dataSources.xml
+.idea/dataSources.local.xml
+.idea/sqlDataSources.xml
+.idea/dynamic.xml
+.idea/uiDesigner.xml
+/.idea/codeStyles/codeStyleConfig.xml

subsurface/interfaces/liquid_earth/rest_client.py

@@ -95,6 +95,11 @@ def add_data_to_project(self, project_id: str, data_name: str, data_type: DataTy
         self._put_file_in_project(project_id, data_name + ".le", data_type, body)
         self._put_file_in_project(project_id, data_name + ".json", data_type,  json.dumps(header))
 
+    def add_texture_to_mesh(self, project_id: str, data_name: str, data_type: DataTypes,
+                            texture):
+        self._post_update_meta_data(project_id, data_name, data_type)
+        self._put_file_in_project(project_id, data_name + ".png", data_type, texture)
+
     def _put_file_in_project(self, project_id: str, data_name, data_type: DataTypes, file):
         blob_path = data_type.value + "/" + data_name
 
@@ -105,9 +110,18 @@ def _put_file_in_project(self, project_id: str, data_name, data_type: DataTypes,
             raise Exception(f"Request failed: {response.text}")
         elif response.status_code >= 200:
             print(response.text)
+        else:
+            print(response.status_code)
 
     def _post_update_meta_data(self, project_id: str, data_name: str, data_type: DataTypes):
-        query_param = f"?project_id={project_id}&data_id={data_name}&data_type={data_type.value}"
+        
+        # ! data_type Collar and tubes should be just mapped to well
+        if data_type == DataTypes.collars or data_type == DataTypes.cylinder:
+            address_in_cosmos = "wells"
+        else:
+            address_in_cosmos = data_type.value
+        
+        query_param = f"?project_id={project_id}&data_id={data_name}&data_type={address_in_cosmos}"
         end_point = "subsurface-lite/v1/update_project_meta" + query_param
 
         response = requests.post(self.host + end_point, headers=self.header)

subsurface/reader/readers_data.py

@@ -6,18 +6,25 @@
 import numpy as np
 import pandas as pd
 import xarray as xr
-from pandas._typing import FilePath, ReadCsvBuffer, ReadCsvBuffer
+
 
 from subsurface.utils.utils_core import get_extension
 
 
 __all__ = ['ReaderFilesHelper', 'ReaderUnstructuredHelper',
            'ReaderWellsHelper', 'RawDataOptions', 'RawDataUnstructured']
 
+if pd.__version__ < '1.4.0':
+    from pandas._typing import FilePathOrBuffer
+    fb = FilePathOrBuffer
+elif pd.__version__ >= '1.4.0':
+    from pandas._typing import FilePath, ReadCsvBuffer
+    fb = Union[FilePath, ReadCsvBuffer[bytes], ReadCsvBuffer[str]]
 
 @dataclass
 class ReaderFilesHelper:
-    file_or_buffer: Union[FilePath, ReadCsvBuffer[bytes], ReadCsvBuffer[str]]
+    file_or_buffer: fb
+
 
     usecols: Union[List[str], List[int]] = None # Use a subset of columns
     col_names: List[Union[str, int]] = None # Give a name

subsurface/reader/topography/topo_core.py

@@ -29,11 +29,12 @@ def read_structured_topography_to_unstructured(path) -> UnstructuredData:
 
 def rasterio_dataset_to_structured_data(dataset):
     data = dataset.read(1)
+    data = np.fliplr(data.T)
     shape = data.shape
     coords = {
         'x': np.linspace(
-            dataset.bounds.right,
             dataset.bounds.left,
+            dataset.bounds.right,
             shape[0]
         ),
         'y': np.linspace(

subsurface/reader/volume/volume_utils.py

@@ -31,7 +31,7 @@ def interpolate_unstructured_data_to_structured_data(
     grid = np.meshgrid(*coords.values())
 
     interpolated_attributes = griddata(ud.vertex,
-                                       ud.points_attributes.loc[:, attr_name],
+                                       ud.attributes.loc[:, attr_name],
                                        tuple(grid), method=interpolation_method.value)
 
     sd = StructuredData.from_numpy(

subsurface/reader/wells/pandas_to_welly.py

@@ -98,7 +98,8 @@ def add_datum(self, data: pd.DataFrame):
             w = self.p.get_well(b)
             datum = data.loc[[b]]
             assert datum.shape[1] == 3, 'datum must be XYZ coord'
-            w.position = datum.values[0]
+
+            w.location.position = datum.values[0]
 
         return self.p
 
@@ -114,8 +115,10 @@ def add_striplog(self, data: pd.DataFrame):
         for b in unique_borehole:
             w = self.p.get_well(b)
             data_dict = data.loc[[b]].to_dict('list')
-
-            s = Striplog.from_dict_advanced(data_dict, points=True)
+            data_csv = data.loc[[b]].to_csv()
+            #s = Striplog.from_dict_advanced(data_dict, points=True)
+            #s = Striplog.from_dict(data_dict)
+            s = Striplog.from_csv(text=data_csv)
 
             try:
                 n_basis = w.location.md.shape[0]

subsurface/reader/wells/well_files_reader.py

@@ -1,6 +1,7 @@
 import warnings
 from typing import Dict
 
+import pandas
 import pandas as pd
 import numpy as np
 
@@ -89,7 +90,7 @@ def read_survey_df_from_las(reader_helper: ReaderFilesHelper, well_name: str) ->
     welly_well = _create_welly_well_from_las(well_name, reader_helper.file_or_buffer)
     survey_df = welly_well.df()[reader_helper.usecols]
     map_rows_and_cols_inplace(survey_df, reader_helper)
-    survey_df["well_name"] = "Cottessen"
+    survey_df["well_name"] = well_name
     survey_df.set_index("well_name", inplace=True)
     return survey_df
 

subsurface/reader/wells/welly_reader.py

@@ -97,7 +97,13 @@ def vertex_and_cells_from_welly_trajectory(cells: np.ndarray, elev: bool,
                                            welly_trajectory_kwargs: dict,
                                            last_index: int, n_vertex_for_well: int,
                                            vertex: np.ndarray, w: Well):
-    xyz = w.location.trajectory(w.position, elev, n_vertex_for_well, **welly_trajectory_kwargs)
+
+    try:
+        datum = w.location.datum
+    except AttributeError:
+        datum = None
+
+    xyz = w.location.trajectory(datum=datum, elev=elev, points=n_vertex_for_well, **welly_trajectory_kwargs) #w.location.position
     # Make sure deviation is there
     a = np.arange(0 + last_index, xyz.shape[0] - 1 + last_index, dtype=np.int_)
     b = np.arange(1 + last_index, xyz.shape[0] + last_index, dtype=np.int_)
@@ -145,6 +151,16 @@ def _create_well(uwi: str = 'dummy_uwi'):
 def add_curves_from_las(w: Well, las_folder: str) -> Well:
     """ Add curves from las file. """
 
+    def _read_curves_to_welly_object(well: Well, curve_path: str = '.') -> Well:
+        las_files = glob.glob(curve_path + '*.las')
+        # throw error if no las files found
+        if len(las_files) == 0:
+            raise Exception('No las files found in ' + curve_path)
+
+        for curve in las_files:
+            well.add_curves_from_las(curve)
+        return well
+
     _read_curves_to_welly_object(w, curve_path=las_folder)
 
     w_df = w.df()
@@ -166,12 +182,4 @@ def _make_deviation_df(well_df: DataFrame, inclination_header: str, azimuth_head
     return deviation_complete
 
 
-def _read_curves_to_welly_object(well: Well, curve_path: str = '.') -> Well:
-    las_files = glob.glob(curve_path + '*.las')
-    # throw error if no las files found
-    if len(las_files) == 0:
-        raise Exception('No las files found in ' + curve_path)
 
-    for curve in las_files:
-        well.add_curves_from_las(curve)
-    return well

subsurface/structs/base_structures/unstructured_data.py

@@ -7,7 +7,6 @@
 
 from subsurface.reader.readers_data import RawDataUnstructured
 
-
 __all__ = ['UnstructuredData', ]
 
 
@@ -119,14 +118,15 @@ def from_data_arrays_dict(cls, xarray_dict: Dict[str, xr.DataArray],
                               default_cells_attributes_name="cell_attrs",
                               default_points_attributes_name="vertex_attrs"):
 
+        # TODO: xr.Dataset seems to have been changed with 2022.06. needs to be adapted for indexing
         ds = xr.Dataset(xarray_dict, coords=coords, attrs=xarray_attributes)
 
         # Try to unstack pandas dataframe if exist
         # TODO: This is an issue in wells. If it is only there maybe we should move it there
         try:
             ds = ds.reset_index('cell')
-        except KeyError:
-            pass
+        except (KeyError, ValueError) as e:
+            print(f"{e} xarray dataset must include 'cell' key (KeyError) or xarray 'cell' has no index (ValueError).")
 
         return cls(ds, default_cells_attributes_name, default_points_attributes_name)
 
@@ -256,7 +256,7 @@ def points_attributes_to_dict(self, orient='list'):
     def extent(self):
         max = self.vertex.max(axis=0)
         min = self.vertex.min(axis=0)
-        extent = np.stack((min, max), axis = 1).ravel()
+        extent = np.stack((min, max), axis=1).ravel()
         return extent
 
     def to_xarray(self):

subsurface/writer/to_binary.py

@@ -2,7 +2,10 @@
 
 
 def base_structs_to_binary_file(path, base_struct, order='F'):
-    bytearray_le, header = base_struct.default_data_array_to_binary(order=order)
+    try:
+        bytearray_le, header = base_struct.default_data_array_to_binary(order=order)
+    except AttributeError:
+        bytearray_le, header = base_struct.to_binary(order=order)
     with open(path+'.json', 'w') as outfile:
         json.dump(header, outfile)
 

tests/data/borehole/wells-database-small.xlsx

@@ -30,8 +30,7 @@ def wells(data_path):
 
 
 def test_wells_to_binary(wells):
-    bytearray_le, header = wells.default_data_array_to_binary()
-    print(header)
+    bytearray_le, header = wells.to_binary()
 
     with open('well_f.json', 'w') as outfile:
         json.dump(header, outfile)
@@ -78,7 +77,7 @@ def test_profile_to_binary(data_path):
     import pandas as pd
 
     unstruct = UnstructuredData.from_array(v, e, vertex_attr=pd.DataFrame(uv, columns=['u', 'v']))
-    mesh_binary, mesh_header = unstruct.default_data_array_to_binary()
+    mesh_binary, mesh_header = unstruct.to_binary()
 
     with open('mesh_uv.json', 'w') as outfile:
         import json

tests/test_interfaces/test_to_binary.py

@@ -19,6 +19,8 @@
 
 pf = pathlib.Path(__file__).parent.absolute()
 data_path = pf.joinpath('../data/borehole/')
+import sys
+sys.path.insert(0, '../../../striplog_miguel/striplog/')
 from striplog import Striplog, Component
 
 
@@ -228,13 +230,19 @@ def test_excel_to_subsurface():
     well_names = data[well_name_column].unique()
 
     foo = data.groupby(well_name_column).get_group(well_names[0])
-    data_dict = foo.to_dict('list')
+    foo.columns = foo.columns.map({'DEPTH_FROM': 'top',
+                               'DEPTH_TO': 'base',
+                               'LITHOLOGY': 'component lith',
+                               'SITE_ID': 'description'})
+    foo_csv = foo.to_csv(index=False)
+    #data_dict = foo.to_dict('list')
 
     # Load striplog
-    s = Striplog.from_dict_advanced(data_dict, remap={'DEPTH_FROM': 'top',
-                                             'DEPTH_TO': 'base',
-                                             'LITHOLOGY': 'component lith',
-                                             'SITE_ID': 'description'})
+    s = Striplog.from_csv(text=foo_csv)
+    # s = Striplog.from_dict_advanced(data_dict, remap={'DEPTH_FROM': 'top',
+    #                                          'DEPTH_TO': 'base',
+    #                                          'LITHOLOGY': 'component lith',
+    #                                          'SITE_ID': 'description'})
 
     s.plot()
     plt.show(block=False)
@@ -272,12 +280,23 @@ def test_striplog_2():
     well_name_column = 'SITE_ID'
     well_names = data[well_name_column].unique()
     foo = data.groupby(well_name_column).get_group(well_names[0])
-    data_dict = foo.to_dict('list')
 
-    s = Striplog.from_dict_advanced(data_dict, remap={'DEPTH_FROM': 'top',
-                                             'DEPTH_TO': 'base',
-                                             'LITHOLOGY': 'component lith',
-                                             'SITE_ID': 'description'})
+    foo.columns = foo.columns.map({'DEPTH_FROM': 'top',
+                                   'DEPTH_TO': 'base',
+                                   'LITHOLOGY': 'component lith',
+                                   'SITE_ID': 'description'})
+    foo_csv = foo.to_csv(index=False)
+    # data_dict = foo.to_dict('list')
+
+    # Load striplog
+    s = Striplog.from_csv(text=foo_csv)
+    # TODO: Maybe get rid of from_dict_advanced, as it does not show a benefit over csv?
+    # data_dict = foo.to_dict('list')
+    #
+    # s = Striplog.from_dict_advanced(data_dict, remap={'DEPTH_FROM': 'top',
+    #                                          'DEPTH_TO': 'base',
+    #                                          'LITHOLOGY': 'component lith',
+    #                                          'SITE_ID': 'description'})
 
     s.plot()
     plt.show(block=False)
@@ -417,7 +436,7 @@ def test_read_kim_default_component_table():
     subsurface.visualization.pv_plot([pyvista_mesh], image_2d=True)
 
 
-def test_read_wells():
+def test_read_wells(): #TODO: fix trajectory IndexError of Well.location.trajectory() Missing NC-10 End, added a dummy
 
     file_name = 'wells-database-small.xlsx'