Merge pull request #114 from ExpediaGroup/feast_merge_q4_2023

chore: Feast merge Q4 2023

.github/workflows/linter.yml

@@ -20,9 +20,6 @@ jobs:
         uses: actions/setup-go@v2
         with:
           go-version: 1.19.7
-      - name: Upgrade pip version
-        run: |
-          pip install --upgrade "pip>=21.3.1,<23.2"
       - name: Get pip cache dir
         id: pip-cache
         run: |

.github/workflows/pr_local_integration_tests.yml

@@ -76,6 +76,7 @@ jobs:
 
       - name: Install dependencies
         run: make install-python-ci-dependencies
+
       - name: Test local integration tests
         if: ${{ always() }}  # this will guarantee that step won't be canceled and resources won't leak
         env:

.github/workflows/release.yml

@@ -36,8 +36,7 @@ jobs:
       - name: Setup Node.js
         uses: actions/setup-node@v3
         with:
-          node-version: '18.x'
-          registry-url: 'https://registry.npmjs.org'
+          node-version: "lts/*"
       - name: Release (Dry Run)
         id: get_versions
         run: |
@@ -60,10 +59,10 @@ jobs:
       NEXT_VERSION: ${{ needs.get_dry_release_versions.outputs.next_version }}
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
+      - name: Setup Node.js
+        uses: actions/setup-node@v3
         with:
-          node-version: '18.x'
-          registry-url: 'https://registry.npmjs.org'
+          node-version: "lts/*"
       - name: Bump file versions
         run: python ./infra/scripts/release/bump_file_versions.py ${CURRENT_VERSION} ${NEXT_VERSION}
       - name: Install yarn dependencies
@@ -101,10 +100,10 @@ jobs:
       NEXT_VERSION: ${{ needs.get_dry_release_versions.outputs.next_version }}
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
+      - name: Setup Node.js
+        uses: actions/setup-node@v3
         with:
-          node-version: '18.x'
-          registry-url: 'https://registry.npmjs.org'
+          node-version: "lts/*"
       - name: Bump file versions (temporarily for Web UI publish)
         run: python ./infra/scripts/release/bump_file_versions.py ${CURRENT_VERSION} ${NEXT_VERSION}
       - name: Install yarn dependencies
@@ -139,8 +138,7 @@ jobs:
     - name: Setup Node.js
       uses: actions/setup-node@v3
       with:
-        node-version: '18.x'
-        registry-url: 'https://registry.npmjs.org'
+        node-version: "lts/*"
     - name: Set up Homebrew
       id: set-up-homebrew
       uses: Homebrew/actions/setup-homebrew@master

.github/workflows/unit_tests.yml

@@ -20,7 +20,6 @@ jobs:
       OS: ${{ matrix.os }}
       PYTHON: ${{ matrix.python-version }}
     steps:
-      - uses: actions/checkout@v3
       - name: Increase swapfile
         # Increase ubuntu's swapfile to avoid running out of resources which causes the action to terminate
         if: startsWith(matrix.os, 'ubuntu')
@@ -31,7 +30,7 @@ jobs:
           sudo mkswap /swapfile
           sudo swapon /swapfile
           sudo swapon --show
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Setup Python
         id: setup-python
         uses: actions/setup-python@v3
@@ -44,13 +43,13 @@ jobs:
         with:
           go-version: 1.19.7
       - name: Install mysql on macOS
-        if:  startsWith(matrix.os, 'macOS')
+        if: startsWith(matrix.os, 'macOS')
         run: |
           brew install mysql
           PATH=$PATH:/usr/local/mysql/bin
       - name: Work around Homebrew MySQL being broken
         # See https://github.com/Homebrew/homebrew-core/issues/130258 for more details.
-        if:  startsWith(matrix.os, 'macOS')
+        if: startsWith(matrix.os, 'macOS')
         run: |
           brew install zlib
           ln -sv $(brew --prefix zlib)/lib/libz.dylib $(brew --prefix)/lib/libzlib.dylib

CHANGELOG.md

@@ -1,5 +1,42 @@
 # Changelog
 
+# [0.35.0](https://github.com/feast-dev/feast/compare/v0.34.0...v0.35.0) (2024-01-13)
+
+
+### Bug Fixes
+
+* Add async refresh to prevent synchronous refresh in main thread ([#3812](https://github.com/feast-dev/feast/issues/3812)) ([9583ed6](https://github.com/feast-dev/feast/commit/9583ed6b4ae8d3b97934bf0c80ecb236ed1e2895))
+* Adopt connection pooling for HBase ([#3793](https://github.com/feast-dev/feast/issues/3793)) ([b3852bf](https://github.com/feast-dev/feast/commit/b3852bfb8b27bf07736935f465da3067fcbac0ae))
+* Bytewax engine create configmap from object ([#3821](https://github.com/feast-dev/feast/issues/3821)) ([25e9775](https://github.com/feast-dev/feast/commit/25e97756adedfd1227d591ae74bdf60655f9067e))
+* Fix warnings from deprecated paths and update default log level ([#3757](https://github.com/feast-dev/feast/issues/3757)) ([68a8737](https://github.com/feast-dev/feast/commit/68a87379c42567f338d86cb2be90520cc6d4bfb6))
+* improve parsing bytewax job status ([5983f40](https://github.com/feast-dev/feast/commit/5983f40f8f5df5dbbcd2640f83ef82c19cdb4d19))
+* make bytewax settings unexposed ([ae1bb8b](https://github.com/feast-dev/feast/commit/ae1bb8bdd1b9e293809519971935c93c2214d791))
+* Make generated temp table name escaped ([#3797](https://github.com/feast-dev/feast/issues/3797)) ([175d796](https://github.com/feast-dev/feast/commit/175d7969b1f75ab797aff9c92a70d845297444ad))
+* Pin numpy version to avoid spammy deprecation messages ([774ed33](https://github.com/feast-dev/feast/commit/774ed33a067bf9bf087520325b72f4f4d194106a))
+* Redundant feature materialization and premature incremental materialization timestamp updates ([#3789](https://github.com/feast-dev/feast/issues/3789)) ([417b16b](https://github.com/feast-dev/feast/commit/417b16b57af7b38fbd0708b9a0c5d5035ed021fd)), closes [#6](https://github.com/feast-dev/feast/issues/6) [#7](https://github.com/feast-dev/feast/issues/7)
+* Resolve hbase hotspot issue when materializing ([#3790](https://github.com/feast-dev/feast/issues/3790)) ([7376db8](https://github.com/feast-dev/feast/commit/7376db8dbd1d3168a1262fbbc0ce3899be8d0c34))
+* Set keepalives_idle None by default ([#3756](https://github.com/feast-dev/feast/issues/3756)) ([8717e9b](https://github.com/feast-dev/feast/commit/8717e9bf0fd253454982b9c9e9527c4d41906e9c))
+* Set upper bound for bigquery client due to its breaking changes ([2151c39](https://github.com/feast-dev/feast/commit/2151c39d1a8d8eba114306411dd4bd91ac0ce3f6))
+* UI project cannot handle fallback routes ([#3766](https://github.com/feast-dev/feast/issues/3766)) ([96ece0f](https://github.com/feast-dev/feast/commit/96ece0fe94a07cc6f1dabf5d6c9b061b48b06d67))
+* update dependencies versions due to conflicts ([5dc0b24](https://github.com/feast-dev/feast/commit/5dc0b241ec68aa10fd783569bf0ae12c5752f20f))
+* Update jackson and remove unnecessary logging ([#3809](https://github.com/feast-dev/feast/issues/3809)) ([018d0ea](https://github.com/feast-dev/feast/commit/018d0eab69dde63266f2c56813045ea5c5523f76))
+* upgrade the pyarrow to latest v14.0.1 for CVE-2023-47248. ([052182b](https://github.com/feast-dev/feast/commit/052182bcca046e35456674fc7d524825882f4b35))
+
+
+### Features
+
+* Add get online feature rpc to gprc server ([#3815](https://github.com/feast-dev/feast/issues/3815)) ([01db8cc](https://github.com/feast-dev/feast/commit/01db8cce6f82d4c6e496041351fb6b56eb2645b0))
+* Add materialize and materialize-incremental rest endpoints ([#3761](https://github.com/feast-dev/feast/issues/3761)) ([fa600fe](https://github.com/feast-dev/feast/commit/fa600fe3c4b1d5fdd383a9367511ac5616ee7a32)), closes [#3760](https://github.com/feast-dev/feast/issues/3760)
+* add redis sentinel support ([3387a15](https://github.com/feast-dev/feast/commit/3387a15d2b7e8dea430a271570be5a19b32bd3fe))
+* add redis sentinel support ([4337c89](https://github.com/feast-dev/feast/commit/4337c89083a3cfca21ee1beef473fda13b0e9014))
+* add redis sentinel support format lint ([aad8718](https://github.com/feast-dev/feast/commit/aad8718d24d893b3ff8c5864c5b8d210cfcdb22f))
+* Add support for `table_create_disposition` in bigquery job for offline store ([#3762](https://github.com/feast-dev/feast/issues/3762)) ([6a728fe](https://github.com/feast-dev/feast/commit/6a728fe66db0286ea10301d1fe693d6dcba4e4f4))
+* Add support for in_cluster config and additional labels for bytewax materialization ([#3754](https://github.com/feast-dev/feast/issues/3754)) ([2192e65](https://github.com/feast-dev/feast/commit/2192e6527fa10f1580e4dd8f350e05e45af981b7))
+* Apply cache to load proto registry for performance ([#3702](https://github.com/feast-dev/feast/issues/3702)) ([709c709](https://github.com/feast-dev/feast/commit/709c7098dc28a35dd488f5079d3787cf1f74ec03))
+* Make bytewax job write as mini-batches ([#3777](https://github.com/feast-dev/feast/issues/3777)) ([9b0e5ce](https://github.com/feast-dev/feast/commit/9b0e5ce2d1b617fcdcf0699c8b0cf8549a5e5ac5))
+* Optimize bytewax pod resource with zero-copy ([9cf9d96](https://github.com/feast-dev/feast/commit/9cf9d965a5566a87bb7419f2e8509666076f035f))
+* Support GCS filesystem for bytewax engine ([#3774](https://github.com/feast-dev/feast/issues/3774)) ([fb6b807](https://github.com/feast-dev/feast/commit/fb6b807f8b32776d388757ca431d290c03170c66))
+
 # [0.34.0](https://github.com/feast-dev/feast/compare/v0.33.0...v0.34.0) (2023-09-07)
 
 

CODEOWNERS

@@ -10,9 +10,14 @@
 # Core utils
 /sdk/python/feast/infra/offline_stores/offline_utils.py @feast-dev/maintainers @sfc-gh-madkins
 
-# BigQuery
+# Offline interfaces
 /sdk/python/feast/infra/offline_stores/offline_store.py @feast-dev/maintainers
 
+# BigQuery
+/sdk/python/feast/infra/offline_stores/bigquery.py @sudohainguyen
+/sdk/python/feast/infra/offline_stores/bigquery_source.py @sudohainguyen
+/sdk/python/tests/integration/feature_repos/universal/data_sources/bigquery.py @sudohainguyen
+
 # Snowflake
 /sdk/python/feast/infra/offline_stores/snowflake* @sfc-gh-madkins
 
@@ -27,6 +32,10 @@
 
 # ==== Online Stores ====
 
+# HBase
+/sdk/python/feast/infra/online_stores/hbase.py @sudohainguyen
+/sdk/python/feast/infra/online_stores/contrib/hbase_online_store     @sudohainguyen
+
 # Redis
 /sdk/python/feast/infra/online_stores/redis.py @DvirDukhan
 /java/feast/serving/connectors/redis/ @DvirDukhan

Makefile

@@ -390,7 +390,6 @@ compile-protos-go: install-go-proto-dependencies install-protoc-dependencies
 
 compile-go-lib: install-go-proto-dependencies install-go-ci-dependencies
 	CGO_LDFLAGS_ALLOW=".*" COMPILE_GO=True python setup.py build_ext --inplace
-	pip install --ignore-installed protobuf==4.23.4 "grpcio-tools>=1.56.2,<2" mypy-protobuf==3.1.0
 
 install-feast-ci-locally:
 	pip install -e ".[ci]"

community/maintainers.md

@@ -9,10 +9,12 @@ In alphabetical order
 | Name           | GitHub Username  | Email                       | Organization       |
 | -------------- | ---------------- |-----------------------------| ------------------ |
 | Achal Shah     | `achals`         | [email protected]            | Tecton             |
-| Felix Wang     | `felixwang9817`  | [email protected]       | Tecton             |
-| Kevin Zhang    | `kevjumba`       | [email protected] | Tecton             |
-| Miles Adkins   | `sfc-gh-madkins` | [email protected]  | Snowflake          |
-| Willem Pienaar | `woop`           | [email protected]      | Tecton             |
+| Edson Tirelli   | `etirelli`       | [email protected]          | Red Hat          |
+| Francisco Javier Arceo   | `franciscojavierarceo`       | [email protected]          | Affirm          |
+| Hao Xu   | `HaoXuAI`       | [email protected]          | JPMorgan          |
+| Jeremy Ary   | `jeremyary`       | [email protected]          | Red Hat          |
+| Shuchu Han  | `shuchu`       | [email protected]          | Independent          |
+| Willem Pienaar | `woop`           | [email protected]      | Cleric             |
 | Zhiling Chen   | `zhilingc`       | [email protected]          | GetGround          |
 
 ## Emeritus Maintainers
@@ -29,3 +31,6 @@ In alphabetical order
 | Danny Chiao         | adchia          | [email protected]             | Tecton            |
 | David Liu           | mavysavydav     | [email protected]      | Twitter           |
 | Matt Delacour       | MattDelac       | [email protected]           | Shopify           |
+| Miles Adkins        | sfc-gh-madkins  | [email protected] | Snowflake        |
+| Felix Wang     | `felixwang9817`  | [email protected]       | Tecton             |
+| Kevin Zhang    | `kevjumba`       | [email protected] | Tecton             |

docs/reference/online-stores/redis.md

@@ -45,6 +45,21 @@ online_store:
 ```
 {% endcode %}
 
+Connecting to a Redis Sentinel with SSL enabled and password authentication:
+
+{% code title="feature_store.yaml" %}
+```yaml
+project: my_feature_repo
+registry: data/registry.db
+provider: local
+online_store:
+  type: redis
+  redis_type: redis_sentinel
+  sentinel_master: mymaster
+  connection_string: "redis1:26379,ssl=true,password=my_password"
+```
+{% endcode %}
+
 Additionally, the redis online store also supports automatically deleting data via a TTL mechanism.
 The TTL is applied at the entity level, so feature values from any associated feature views for an entity are removed together. 
 This TTL can be set in the `feature_store.yaml`, using the `key_ttl_seconds` field in the online store. For example:

infra/charts/feast-feature-server/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: feast-feature-server
 description: Feast Feature Server in Go or Python
 type: application
-version: 0.34.0
+version: 0.35.0
 keywords:
   - machine learning
   - big data

infra/charts/feast-feature-server/README.md

@@ -1,6 +1,6 @@
 # Feast Python / Go Feature Server Helm Charts
 
-Current chart version is `0.34.0`
+Current chart version is `0.35.0`
 
 ## Installation
 
@@ -30,7 +30,7 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/python-helm-d
 | fullnameOverride | string | `""` |  |
 | image.pullPolicy | string | `"IfNotPresent"` |  |
 | image.repository | string | `"feastdev/feature-server"` | Docker image for Feature Server repository |
-| image.tag | string | `"0.34.0"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) |
+| image.tag | string | `"0.35.0"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) |
 | imagePullSecrets | list | `[]` |  |
 | livenessProbe.initialDelaySeconds | int | `30` |  |
 | livenessProbe.periodSeconds | int | `30` |  |

infra/charts/feast-feature-server/values.yaml

@@ -9,7 +9,7 @@ image:
   repository: feastdev/feature-server
   pullPolicy: IfNotPresent
   # image.tag -- The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms)
-  tag: 0.34.0
+  tag: 0.35.0
 
 imagePullSecrets: []
 nameOverride: ""

infra/charts/feast/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 description: Feature store for machine learning
 name: feast
-version: 0.34.0
+version: 0.35.0
 keywords:
   - machine learning
   - big data

infra/charts/feast/README.md

@@ -8,7 +8,7 @@ This repo contains Helm charts for Feast Java components that are being installe
 
 ## Chart: Feast
 
-Feature store for machine learning Current chart version is `0.34.0`
+Feature store for machine learning Current chart version is `0.35.0`
 
 ## Installation
 
@@ -65,8 +65,8 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/java-demo) fo
 | Repository | Name | Version |
 |------------|------|---------|
 | https://charts.helm.sh/stable | redis | 10.5.6 |
-| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.34.0 |
-| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.34.0 |
+| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.35.0 |
+| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.35.0 |
 
 ## Values
 

infra/charts/feast/charts/feature-server/Chart.yaml

@@ -1,8 +1,8 @@
 apiVersion: v1
 description: "Feast Feature Server: Online feature serving service for Feast"
 name: feature-server
-version: 0.34.0
-appVersion: v0.34.0
+version: 0.35.0
+appVersion: v0.35.0
 keywords:
 - machine learning
 - big data

infra/charts/feast/charts/feature-server/README.md

@@ -1,6 +1,6 @@
 # feature-server
 
-![Version: 0.34.0](https://img.shields.io/badge/Version-0.34.0-informational?style=flat-square) ![AppVersion: v0.34.0](https://img.shields.io/badge/AppVersion-v0.34.0-informational?style=flat-square)
+![Version: 0.35.0](https://img.shields.io/badge/Version-0.35.0-informational?style=flat-square) ![AppVersion: v0.35.0](https://img.shields.io/badge/AppVersion-v0.35.0-informational?style=flat-square)
 
 Feast Feature Server: Online feature serving service for Feast
 
@@ -17,7 +17,7 @@ Feast Feature Server: Online feature serving service for Feast
 | envOverrides | object | `{}` | Extra environment variables to set |
 | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy |
 | image.repository | string | `"feastdev/feature-server-java"` | Docker image for Feature Server repository |
-| image.tag | string | `"0.34.0"` | Image tag |
+| image.tag | string | `"0.35.0"` | Image tag |
 | ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress |
 | ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth |
 | ingress.grpc.class | string | `"nginx"` | Which ingress controller to use |
@@ -64,4 +64,4 @@ Feast Feature Server: Online feature serving service for Feast
 | transformationService.port | int | `6566` |  |
 
 ----------------------------------------------
-Autogenerated from chart metadata using [helm-docs v1.11.0](https://github.com/norwoodj/helm-docs/releases/v1.11.0)
+Autogenerated from chart metadata using [helm-docs v1.12.0](https://github.com/norwoodj/helm-docs/releases/v1.12.0)

infra/charts/feast/charts/feature-server/values.yaml

@@ -5,7 +5,7 @@ image:
   # image.repository -- Docker image for Feature Server repository
   repository: feastdev/feature-server-java
   # image.tag -- Image tag
-  tag: 0.34.0
+  tag: 0.35.0
   # image.pullPolicy -- Image pull policy
   pullPolicy: IfNotPresent
 

infra/charts/feast/charts/transformation-service/Chart.yaml

@@ -1,8 +1,8 @@
 apiVersion: v1
 description: "Transformation service: to compute on-demand features"
 name: transformation-service
-version: 0.34.0
-appVersion: v0.34.0
+version: 0.35.0
+appVersion: v0.35.0
 keywords:
 - machine learning
 - big data

infra/charts/feast/charts/transformation-service/README.md

@@ -1,6 +1,6 @@
 # transformation-service
 
-![Version: 0.34.0](https://img.shields.io/badge/Version-0.34.0-informational?style=flat-square) ![AppVersion: v0.34.0](https://img.shields.io/badge/AppVersion-v0.34.0-informational?style=flat-square)
+![Version: 0.35.0](https://img.shields.io/badge/Version-0.35.0-informational?style=flat-square) ![AppVersion: v0.35.0](https://img.shields.io/badge/AppVersion-v0.35.0-informational?style=flat-square)
 
 Transformation service: to compute on-demand features
 
@@ -13,7 +13,7 @@ Transformation service: to compute on-demand features
 | envOverrides | object | `{}` | Extra environment variables to set |
 | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy |
 | image.repository | string | `"feastdev/feature-transformation-server"` | Docker image for Transformation Server repository |
-| image.tag | string | `"0.34.0"` | Image tag |
+| image.tag | string | `"0.35.0"` | Image tag |
 | nodeSelector | object | `{}` | Node labels for pod assignment |
 | podLabels | object | `{}` | Labels to be added to Feast Serving pods |
 | replicaCount | int | `1` | Number of pods that will be created |
@@ -25,4 +25,4 @@ Transformation service: to compute on-demand features
 | service.type | string | `"ClusterIP"` | Kubernetes service type |
 
 ----------------------------------------------
-Autogenerated from chart metadata using [helm-docs v1.11.0](https://github.com/norwoodj/helm-docs/releases/v1.11.0)
+Autogenerated from chart metadata using [helm-docs v1.12.0](https://github.com/norwoodj/helm-docs/releases/v1.12.0)

infra/charts/feast/charts/transformation-service/values.yaml

@@ -5,7 +5,7 @@ image:
   # image.repository -- Docker image for Transformation Server repository
   repository: feastdev/feature-transformation-server
   # image.tag -- Image tag
-  tag: 0.34.0
+  tag: 0.35.0
   # image.pullPolicy -- Image pull policy
   pullPolicy: IfNotPresent
 

infra/charts/feast/requirements.yaml

@@ -1,12 +1,12 @@
 dependencies:
 - name: feature-server
   alias: feature-server
-  version: 0.34.0
+  version: 0.35.0
   condition: feature-server.enabled
   repository: https://feast-helm-charts.storage.googleapis.com
 - name: transformation-service
   alias: transformation-service
-  version: 0.34.0
+  version: 0.35.0
   condition: transformation-service.enabled
   repository: https://feast-helm-charts.storage.googleapis.com
 - name: redis