Skip to content

Commit eb58e01

Browse files
felixwang9817felixwang9817
authored
Merge pull request #40 from felixwang9817/support_on_demand_transforms_with_python_fts
Add support for on demand transforms using the Python FTS
2 parents 1550442 + 06f3f12 commit eb58e01

17 files changed

Lines changed: 843 additions & 19 deletions

‎.gitmodules‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
[submodule "deps/feast"]
22
path = deps/feast
33
url = https://github.com/feast-dev/feast
4-
branch = v0.9-branch
4+
branch = master

‎common/src/main/java/feast/common/models/FeatureV2.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,4 +34,17 @@ public static String getFeatureStringRef(FeatureReferenceV2 featureReference) {
3434
}
3535
return ref;
3636
}
37+
38+
/**
39+
* Accepts either a feature reference of the form "featuretable_name:feature_name" or just a
40+
* feature name, and returns just the feature name. For example, given either
41+
* "driver_hourly_stats:conv_rate" or "conv_rate", "conv_rate" would be returned.
42+
*
43+
* @param featureReference {String}
44+
* @return Base feature name of the feature reference
45+
*/
46+
public static String getFeatureName(String featureReference) {
47+
String[] tokens = featureReference.split(":", 2);
48+
return tokens[tokens.length - 1];
49+
}
3750
}

‎deps/feast‎

Submodule feast updated 194 files

‎pom.xml‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -269,7 +269,38 @@
269269
<version>${grpc.version}</version>
270270
<scope>test</scope>
271271
</dependency>
272-
272+
273+
<!-- https://mvnrepository.com/artifact/org.apache.arrow/arrow-java-root -->
274+
<dependency>
275+
<groupId>org.apache.arrow</groupId>
276+
<artifactId>arrow-java-root</artifactId>
277+
<version>5.0.0</version>
278+
<type>pom</type>
279+
</dependency>
280+
281+
<!-- https://mvnrepository.com/artifact/org.apache.arrow/arrow-vector -->
282+
<dependency>
283+
<groupId>org.apache.arrow</groupId>
284+
<artifactId>arrow-vector</artifactId>
285+
<version>5.0.0</version>
286+
</dependency>
287+
288+
<!-- https://mvnrepository.com/artifact/org.apache.arrow/arrow-memory -->
289+
<dependency>
290+
<groupId>org.apache.arrow</groupId>
291+
<artifactId>arrow-memory</artifactId>
292+
<version>5.0.0</version>
293+
<type>pom</type>
294+
</dependency>
295+
296+
<!-- https://mvnrepository.com/artifact/org.apache.arrow/arrow-memory-netty -->
297+
<dependency>
298+
<groupId>org.apache.arrow</groupId>
299+
<artifactId>arrow-memory-netty</artifactId>
300+
<version>5.0.0</version>
301+
<scope>runtime</scope>
302+
</dependency>
303+
273304
<!-- HTTP/REST -->
274305
<dependency>
275306
<groupId>io.swagger</groupId>

‎serving/pom.xml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -269,6 +269,37 @@
269269
<version>1.10.2</version>
270270
</dependency>
271271

272+
<!-- https://mvnrepository.com/artifact/org.apache.arrow/arrow-java-root -->
273+
<dependency>
274+
<groupId>org.apache.arrow</groupId>
275+
<artifactId>arrow-java-root</artifactId>
276+
<version>5.0.0</version>
277+
<type>pom</type>
278+
</dependency>
279+
280+
<!-- https://mvnrepository.com/artifact/org.apache.arrow/arrow-vector -->
281+
<dependency>
282+
<groupId>org.apache.arrow</groupId>
283+
<artifactId>arrow-vector</artifactId>
284+
<version>5.0.0</version>
285+
</dependency>
286+
287+
<!-- https://mvnrepository.com/artifact/org.apache.arrow/arrow-memory -->
288+
<dependency>
289+
<groupId>org.apache.arrow</groupId>
290+
<artifactId>arrow-memory</artifactId>
291+
<version>5.0.0</version>
292+
<type>pom</type>
293+
</dependency>
294+
295+
<!-- https://mvnrepository.com/artifact/org.apache.arrow/arrow-memory-netty -->
296+
<dependency>
297+
<groupId>org.apache.arrow</groupId>
298+
<artifactId>arrow-memory-netty</artifactId>
299+
<version>5.0.0</version>
300+
<scope>runtime</scope>
301+
</dependency>
302+
272303
<!-- Utilities -->
273304
<dependency>
274305
<groupId>com.fasterxml.jackson.dataformat</groupId>

‎serving/src/main/java/feast/serving/config/FeastProperties.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,16 @@ public void setRegistry(final String registry) {
8282
this.registry = registry;
8383
}
8484

85+
private String transformationServiceEndpoint;
86+
87+
public String getTransformationServiceEndpoint() {
88+
return transformationServiceEndpoint;
89+
}
90+
91+
public void setTransformationServiceEndpoint(final String transformationServiceEndpoint) {
92+
this.transformationServiceEndpoint = transformationServiceEndpoint;
93+
}
94+
8595
private CoreAuthenticationProperties coreAuthentication;
8696

8797
public CoreAuthenticationProperties getCoreAuthentication() {

‎serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import com.google.protobuf.AbstractMessageLite;
2424
import feast.serving.registry.LocalRegistryRepo;
2525
import feast.serving.service.OnlineServingServiceV2;
26+
import feast.serving.service.OnlineTransformationService;
2627
import feast.serving.service.ServingServiceV2;
2728
import feast.serving.specs.CachedSpecService;
2829
import feast.serving.specs.CoreFeatureSpecRetriever;
@@ -126,7 +127,13 @@ public ServingServiceV2 servingServiceV2(
126127
log.info("Created CoreFeatureSpecRetriever");
127128
featureSpecRetriever = new CoreFeatureSpecRetriever(specService);
128129

129-
servingService = new OnlineServingServiceV2(retrieverV2, tracer, featureSpecRetriever);
130+
final String transformationServiceEndpoint = feastProperties.getTransformationServiceEndpoint();
131+
final OnlineTransformationService onlineTransformationService =
132+
new OnlineTransformationService(transformationServiceEndpoint, featureSpecRetriever);
133+
134+
servingService =
135+
new OnlineServingServiceV2(
136+
retrieverV2, tracer, featureSpecRetriever, onlineTransformationService);
130137

131138
return servingService;
132139
}
@@ -164,7 +171,13 @@ public ServingServiceV2 registryBasedServingServiceV2(
164171
final LocalRegistryRepo repo = new LocalRegistryRepo(Paths.get(feastProperties.getRegistry()));
165172
featureSpecRetriever = new RegistryFeatureSpecRetriever(repo);
166173

167-
servingService = new OnlineServingServiceV2(retrieverV2, tracer, featureSpecRetriever);
174+
final String transformationServiceEndpoint = feastProperties.getTransformationServiceEndpoint();
175+
final OnlineTransformationService onlineTransformationService =
176+
new OnlineTransformationService(transformationServiceEndpoint, featureSpecRetriever);
177+
178+
servingService =
179+
new OnlineServingServiceV2(
180+
retrieverV2, tracer, featureSpecRetriever, onlineTransformationService);
168181

169182
return servingService;
170183
}

‎serving/src/main/java/feast/serving/registry/LocalRegistryRepo.java‎

Lines changed: 50 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,22 @@
1818

1919
import feast.proto.core.FeatureProto;
2020
import feast.proto.core.FeatureViewProto;
21+
import feast.proto.core.OnDemandFeatureViewProto;
2122
import feast.proto.core.RegistryProto;
2223
import feast.proto.serving.ServingAPIProto;
2324
import feast.serving.exception.SpecRetrievalException;
2425
import java.nio.file.Files;
2526
import java.nio.file.Path;
27+
import java.util.List;
28+
import java.util.Map;
29+
import java.util.function.Function;
30+
import java.util.stream.Collectors;
2631

2732
public class LocalRegistryRepo implements RegistryRepository {
2833
private final RegistryProto.Registry registry;
34+
private Map<String, FeatureViewProto.FeatureViewSpec> featureViewNameToSpec;
35+
private Map<String, OnDemandFeatureViewProto.OnDemandFeatureViewSpec>
36+
onDemandFeatureViewNameToSpec;
2937

3038
public LocalRegistryRepo(Path localRegistryPath) {
3139
if (!localRegistryPath.toFile().exists()) {
@@ -38,6 +46,26 @@ public LocalRegistryRepo(Path localRegistryPath) {
3846
} catch (final Exception e) {
3947
throw new RuntimeException(e);
4048
}
49+
50+
final RegistryProto.Registry registry = this.getRegistry();
51+
List<FeatureViewProto.FeatureViewSpec> featureViewSpecs =
52+
registry.getFeatureViewsList().stream()
53+
.map(fv -> fv.getSpec())
54+
.collect(Collectors.toList());
55+
featureViewNameToSpec =
56+
featureViewSpecs.stream()
57+
.collect(
58+
Collectors.toMap(FeatureViewProto.FeatureViewSpec::getName, Function.identity()));
59+
List<OnDemandFeatureViewProto.OnDemandFeatureViewSpec> onDemandFeatureViewSpecs =
60+
registry.getOnDemandFeatureViewsList().stream()
61+
.map(odfv -> odfv.getSpec())
62+
.collect(Collectors.toList());
63+
onDemandFeatureViewNameToSpec =
64+
onDemandFeatureViewSpecs.stream()
65+
.collect(
66+
Collectors.toMap(
67+
OnDemandFeatureViewProto.OnDemandFeatureViewSpec::getName,
68+
Function.identity()));
4169
}
4270

4371
@Override
@@ -48,15 +76,12 @@ public RegistryProto.Registry getRegistry() {
4876
@Override
4977
public FeatureViewProto.FeatureViewSpec getFeatureViewSpec(
5078
String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) {
51-
final RegistryProto.Registry registry = this.getRegistry();
52-
for (final FeatureViewProto.FeatureView featureView : registry.getFeatureViewsList()) {
53-
if (featureView.getSpec().getName().equals(featureReference.getFeatureTable())) {
54-
return featureView.getSpec();
55-
}
79+
String featureViewName = featureReference.getFeatureTable();
80+
if (featureViewNameToSpec.containsKey(featureViewName)) {
81+
return featureViewNameToSpec.get(featureViewName);
5682
}
5783
throw new SpecRetrievalException(
58-
String.format(
59-
"Unable to find feature view with name: %s", featureReference.getFeatureTable()));
84+
String.format("Unable to find feature view with name: %s", featureViewName));
6085
}
6186

6287
@Override
@@ -75,4 +100,22 @@ public FeatureProto.FeatureSpecV2 getFeatureSpec(
75100
"Unable to find feature with name: %s in feature view: %s",
76101
featureReference.getName(), featureReference.getFeatureTable()));
77102
}
103+
104+
@Override
105+
public OnDemandFeatureViewProto.OnDemandFeatureViewSpec getOnDemandFeatureViewSpec(
106+
String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) {
107+
String onDemandFeatureViewName = featureReference.getFeatureTable();
108+
if (onDemandFeatureViewNameToSpec.containsKey(onDemandFeatureViewName)) {
109+
return onDemandFeatureViewNameToSpec.get(onDemandFeatureViewName);
110+
}
111+
throw new SpecRetrievalException(
112+
String.format(
113+
"Unable to find on demand feature view with name: %s", onDemandFeatureViewName));
114+
}
115+
116+
@Override
117+
public boolean isOnDemandFeatureReference(ServingAPIProto.FeatureReferenceV2 featureReference) {
118+
String onDemandFeatureViewName = featureReference.getFeatureTable();
119+
return onDemandFeatureViewNameToSpec.containsKey(onDemandFeatureViewName);
120+
}
78121
}

‎serving/src/main/java/feast/serving/registry/RegistryRepository.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
import feast.proto.core.FeatureProto;
2020
import feast.proto.core.FeatureViewProto;
21+
import feast.proto.core.OnDemandFeatureViewProto;
2122
import feast.proto.core.RegistryProto;
2223
import feast.proto.serving.ServingAPIProto;
2324

@@ -33,4 +34,9 @@ FeatureViewProto.FeatureViewSpec getFeatureViewSpec(
3334

3435
FeatureProto.FeatureSpecV2 getFeatureSpec(
3536
String projectName, ServingAPIProto.FeatureReferenceV2 featureReference);
37+
38+
OnDemandFeatureViewProto.OnDemandFeatureViewSpec getOnDemandFeatureViewSpec(
39+
String projectName, ServingAPIProto.FeatureReferenceV2 featureReference);
40+
41+
boolean isOnDemandFeatureReference(ServingAPIProto.FeatureReferenceV2 featureReference);
3642
}

0 commit comments

Comments
 (0)