Skip to content

Commit 7609d10

Browse files
rsendenrsenden
committed
feat: Add support for authenticated URL access (closes #1009)
1 parent 708cd54 commit 7609d10

16 files changed

Lines changed: 395 additions & 14 deletions

File tree

‎fcli-core/fcli-app/src/main/resources/com/fortify/cli/app/i18n/FortifyCLIMessages.properties‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,17 +31,17 @@ log-mask = Log mask level: ${COMPLETION-CANDIDATES}. Default: ${DEFAULT-VALUE}.
3131
debug = Enable collection of debug logs.
3232

3333
fcli.action.nameOrLocation = The action to load; either simple name or local or remote action \
34-
YAML file location.
34+
YAML file location. Supports authenticated HTTP(S) URLs.
3535
fcli.action.asciidoc.manpage-dir = Optional directory to write output. If directory contains fcli \
3636
manual pages, any (full) fcli commands in the generated documentation will link to the corresponding \
3737
fcli manual page.
3838

3939
fcli.action.run.action-parameter = Action parameter(s); see 'help' command output to \
4040
list supported parameters.
4141
fcli.action.import.zip = Zip-file containing actions to be imported; may be specified as a path to \
42-
a local zip-file or a URL. Action names will be based on filenames contained in the zip-file.
42+
a local zip-file or a URL. Supports authenticated HTTP(S) URLs. Action names will be based on filenames contained in the zip-file.
4343
fcli.action.import.file = Single action YAML file to be imported; may be specified as a path to a \
44-
local file or a URL. Action name will be based on the given filename.
44+
local file or a URL. Supports authenticated HTTP(S) URLs. Action name will be based on the given filename.
4545
fcli.action.sign.in = Action YAML file to sign.
4646
fcli.action.sign.out = Signed action output file.
4747
fcli.action.sign.with = PEM file containing the private key used for signing. The private key must \
@@ -58,7 +58,7 @@ fcli.action.sign.pubout = Optional public key output file. Use this to extract t
5858
from the private key for distribution.
5959
fcli.action.resolver.from-zip = Optional local or remote zip-file from which to load the action if \
6060
the action is specified as a simple name. For commands that take an action as input (like get, help \
61-
or run), this option will be ignored if action is specified as local or remote action YAML file location.
61+
or run), this option will be ignored if action is specified as local or remote action YAML file location. Supports authenticated HTTP(S) URLs.
6262
fcli.action.resolver.pubkey = Optional public key to use for verifying action signature. Can \
6363
be specified as one of: \
6464
%n file:<local file>%n url:<url>%n string:<string value>%n env:<env-var name>\
@@ -70,7 +70,7 @@ fcli.action.resolver.pubkey = Optional public key to use for verifying action si
7070
example. Note that the given public key will be ignored if its fingerprint doesn't match \
7171
the public key fingerprint stored in the action signature. If no (matching) public key is \
7272
provided, action signature will be verified against public keys previously imported through \
73-
the 'fcli config public-key import' command.
73+
the 'fcli config public-key import' command. Supports authenticated HTTP(S) URLs.
7474
fcli.action.on-invalid-signature = Action to take if action signature is invalid. Allowed values: ${COMPLETION-CANDIDATES}. Default value: ${DEFAULT-VALUE}.
7575
fcli.action.on-unsigned = Action to take if action isn't signed. Allowed values: ${COMPLETION-CANDIDATES}. Default value: ${DEFAULT-VALUE}.
7676
fcli.action.on-no-public-key = Action to take if no matching public key was found. Allowed values: ${COMPLETION-CANDIDATES}. Default value: ${DEFAULT-VALUE}.

‎fcli-core/fcli-common-action/src/main/java/com/fortify/cli/common/action/cli/mixin/ActionResolverMixin.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@
1717
import com.fortify.cli.common.action.helper.ActionLoaderHelper.ActionValidationHandler;
1818
import com.fortify.cli.common.action.model.Action;
1919
import com.fortify.cli.common.cli.mixin.CommonOptionMixins.AbstractTextResolverMixin;
20+
import com.fortify.cli.common.log.LogSensitivityLevel;
21+
import com.fortify.cli.common.log.MaskValue;
22+
import com.fortify.cli.common.rest.unirest.RemoteUrlAuthHelper;
2023

2124
import lombok.Getter;
2225
import picocli.CommandLine.Mixin;
@@ -47,14 +50,17 @@ public String loadActionContents(String type, ActionValidationHandler actionVali
4750
}
4851

4952
public static class RequiredParameter extends AbstractActionResolverMixin {
53+
@MaskValue(sensitivity = LogSensitivityLevel.high, description = "REMOTE URL AUTH VALUE", pattern = RemoteUrlAuthHelper.URL_USERINFO_AUTH_VALUE_MASK_PATTERN)
5054
@Getter @Parameters(arity="1", descriptionKey="fcli.action.nameOrLocation") private String action;
5155
}
5256

5357
public static class OptionalParameter extends AbstractActionResolverMixin {
58+
@MaskValue(sensitivity = LogSensitivityLevel.high, description = "REMOTE URL AUTH VALUE", pattern = RemoteUrlAuthHelper.URL_USERINFO_AUTH_VALUE_MASK_PATTERN)
5459
@Getter @Parameters(arity="0..1", descriptionKey="fcli.action.nameOrLocation") private String action;
5560
}
5661

5762
private static class PublicKeyResolverMixin extends AbstractTextResolverMixin {
63+
@MaskValue(sensitivity = LogSensitivityLevel.high, description = "REMOTE URL AUTH VALUE", pattern = RemoteUrlAuthHelper.URL_USERINFO_AUTH_VALUE_MASK_PATTERN)
5864
@Getter @Option(names={"--pubkey"}, required = false, descriptionKey = "fcli.action.resolver.pubkey", paramLabel = "source") private String textSource;
5965
}
6066
}

‎fcli-core/fcli-common-action/src/main/java/com/fortify/cli/common/action/cli/mixin/ActionSourceResolverMixin.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@
1717
import org.apache.commons.lang3.StringUtils;
1818

1919
import com.fortify.cli.common.action.helper.ActionLoaderHelper.ActionSource;
20+
import com.fortify.cli.common.log.LogSensitivityLevel;
21+
import com.fortify.cli.common.log.MaskValue;
22+
import com.fortify.cli.common.rest.unirest.RemoteUrlAuthHelper;
2023

2124
import lombok.Getter;
2225
import picocli.CommandLine.Option;
@@ -34,6 +37,7 @@ public List<ActionSource> getActionSources(String type) {
3437
}
3538

3639
public static class OptionalOption extends AbstractActionSourceResolverMixin {
40+
@MaskValue(sensitivity = LogSensitivityLevel.high, description = "REMOTE URL AUTH VALUE", pattern = RemoteUrlAuthHelper.URL_USERINFO_AUTH_VALUE_MASK_PATTERN)
3741
@Option(names={"--from-zip", "-z"}, required = false, descriptionKey = "fcli.action.resolver.from-zip")
3842
@Getter private String source;
3943
}

‎fcli-core/fcli-common-action/src/main/java/com/fortify/cli/common/action/helper/ActionImportHelper.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import com.fortify.cli.common.action.model.Action.ActionMetadata;
3737
import com.fortify.cli.common.json.JsonHelper;
3838
import com.fortify.cli.common.output.transform.IActionCommandResultSupplier;
39+
import com.fortify.cli.common.rest.unirest.RemoteUrlAuthHelper;
3940
import com.fortify.cli.common.util.Break;
4041
import com.fortify.cli.common.util.ZipHelper;
4142

@@ -67,7 +68,7 @@ public static ArrayNode importZip(String type, String zip, ActionValidationHandl
6768
@SneakyThrows
6869
private static final InputStream createZipFileInputStream(String zip) {
6970
try {
70-
return new URL(zip).openStream();
71+
return RemoteUrlAuthHelper.openStream(zip);
7172
} catch ( MalformedURLException e ) {
7273
return Files.newInputStream(Path.of(zip));
7374
}

‎fcli-core/fcli-common-action/src/main/java/com/fortify/cli/common/action/helper/ActionLoaderHelper.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515
import java.io.IOException;
1616
import java.io.InputStream;
1717
import java.net.MalformedURLException;
18-
import java.net.URL;
1918
import java.nio.charset.StandardCharsets;
2019
import java.nio.file.Files;
2120
import java.nio.file.Path;
@@ -55,6 +54,7 @@
5554
import com.fortify.cli.common.crypto.helper.impl.SignedTextReader;
5655
import com.fortify.cli.common.exception.FcliBugException;
5756
import com.fortify.cli.common.exception.FcliSimpleException;
57+
import com.fortify.cli.common.rest.unirest.RemoteUrlAuthHelper;
5858
import com.fortify.cli.common.spel.wrapper.TemplateExpressionKeyDeserializer;
5959
import com.fortify.cli.common.util.Break;
6060
import com.fortify.cli.common.util.FcliBuildProperties;
@@ -431,7 +431,7 @@ private static final String commonActionsResourceZip() {
431431
@SneakyThrows
432432
private static final InputStream createSourceInputStream(String source, boolean failOnError) {
433433
try {
434-
return new URL(source).openStream();
434+
return RemoteUrlAuthHelper.openStream(source);
435435
} catch (MalformedURLException mue ) {
436436
try {
437437
return Files.newInputStream(Path.of(source));

‎fcli-core/fcli-common-core/src/main/java/com/fortify/cli/common/cli/mixin/CommonOptionMixins.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@
1313
package com.fortify.cli.common.cli.mixin;
1414

1515
import java.io.File;
16-
import java.net.URL;
1716
import java.nio.charset.StandardCharsets;
1817
import java.nio.file.Files;
1918
import java.nio.file.Path;
@@ -28,6 +27,7 @@
2827
import com.fortify.cli.common.log.LogMaskHelper;
2928
import com.fortify.cli.common.log.LogMaskSource;
3029
import com.fortify.cli.common.log.MaskValue;
30+
import com.fortify.cli.common.rest.unirest.RemoteUrlAuthHelper;
3131
import com.fortify.cli.common.util.EnvHelper;
3232

3333
import lombok.Getter;
@@ -149,7 +149,9 @@ private static final String resolveFile(String file) {
149149

150150
@SneakyThrows
151151
private static final String resolveUrl(String url) {
152-
return IOUtils.toString(new URL(url), StandardCharsets.US_ASCII);
152+
try ( var is = RemoteUrlAuthHelper.openStream(url) ) {
153+
return IOUtils.toString(is, StandardCharsets.US_ASCII);
154+
}
153155
}
154156

155157
private static final String resolveString(String string) {

‎fcli-core/fcli-common-core/src/main/java/com/fortify/cli/common/cli/util/FcliExecutionStrategy.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,13 @@ private static void registerLogMasks(CommandSpec commandSpec) {
161161
.ifPresent(field->registerLogMask(field, value));
162162
}
163163
}
164+
for ( var positionalParameter : commandSpec.positionalParameters() ) {
165+
var value = positionalParameter.getValue();
166+
if ( value!=null ) {
167+
JavaHelper.as(positionalParameter.userObject(), Field.class)
168+
.ifPresent(field->registerLogMask(field, value));
169+
}
170+
}
164171
}
165172

166173
private static void registerLogMask(Field field, Object value) {

‎fcli-core/fcli-common-core/src/main/java/com/fortify/cli/common/rest/unirest/RemoteUrlAuthHelper.java‎

Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
/*
2+
* Copyright 2021-2026 Open Text.
3+
*
4+
* The only warranties for products and services of Open Text
5+
* and its affiliates and licensors ("Open Text") are as may
6+
* be set forth in the express warranty statements accompanying
7+
* such products and services. Nothing herein should be construed
8+
* as constituting an additional warranty. Open Text shall not be
9+
* liable for technical or editorial errors or omissions contained
10+
* herein. The information contained herein is subject to change
11+
* without notice.
12+
*/
13+
package com.fortify.cli.common.rest.unirest;
14+
15+
import java.io.IOException;
16+
import java.io.InputStream;
17+
import java.net.HttpURLConnection;
18+
import java.net.MalformedURLException;
19+
import java.net.URI;
20+
import java.net.URISyntaxException;
21+
import java.net.URL;
22+
import java.net.URLDecoder;
23+
import java.nio.charset.StandardCharsets;
24+
import java.util.Base64;
25+
import java.util.Collections;
26+
import java.util.LinkedHashMap;
27+
import java.util.Map;
28+
29+
import org.apache.commons.lang3.StringUtils;
30+
import org.slf4j.Logger;
31+
import org.slf4j.LoggerFactory;
32+
33+
import com.fortify.cli.common.log.LogMaskHelper;
34+
import com.fortify.cli.common.log.LogMaskSource;
35+
import com.fortify.cli.common.log.LogSensitivityLevel;
36+
37+
public final class RemoteUrlAuthHelper {
38+
private static final Logger LOG = LoggerFactory.getLogger(RemoteUrlAuthHelper.class);
39+
/**
40+
* Pattern for masking URL userinfo auth values through {@link com.fortify.cli.common.log.MaskValue}.
41+
* Captures password for basic auth URLs and token/header value payload for bearer/header(s) formats.
42+
*/
43+
public static final String URL_USERINFO_AUTH_VALUE_MASK_PATTERN = "https?://(?:[^:@/]+:|bearer:|headers?:)([^@]*)@.*";
44+
45+
private static final String PREFIX_BEARER = "bearer:";
46+
private static final String PREFIX_HEADER = "header:";
47+
private static final String PREFIX_HEADERS = "headers:";
48+
49+
private RemoteUrlAuthHelper() {}
50+
51+
public static ParsedRemoteUrl parse(String source) throws MalformedURLException {
52+
var url = new URL(source);
53+
var protocol = url.getProtocol();
54+
if ( !"http".equalsIgnoreCase(protocol) && !"https".equalsIgnoreCase(protocol) ) {
55+
return new ParsedRemoteUrl(source, Collections.emptyMap());
56+
}
57+
58+
try {
59+
var uri = url.toURI();
60+
var headers = parseHeaders(uri.getRawUserInfo());
61+
return new ParsedRemoteUrl(removeUserInfo(uri), headers);
62+
} catch ( URISyntaxException | IllegalArgumentException e ) {
63+
var mue = new MalformedURLException("Invalid URL: "+source);
64+
mue.initCause(e);
65+
throw mue;
66+
}
67+
}
68+
69+
public static InputStream openStream(String source) throws IOException {
70+
var parsed = parse(source);
71+
var url = new URL(parsed.getRequestUrl());
72+
if ( parsed.getHeaders().isEmpty() || !isHttpProtocol(url.getProtocol()) ) {
73+
return url.openStream();
74+
}
75+
76+
LOG.debug("Opening URL: {}", parsed.getRequestUrl());
77+
LogMaskHelper.INSTANCE.registerValue(LogSensitivityLevel.high, LogMaskSource.HTTP_AUTH_HEADER, "REQUEST HEADER", parsed.getHeaders().values(), "");
78+
LOG.debug("Request headers: {}", parsed.getHeaders());
79+
80+
var connection = (HttpURLConnection)url.openConnection();
81+
parsed.getHeaders().forEach(connection::setRequestProperty);
82+
LOG.debug("Response status: {} {}", connection.getResponseCode(), connection.getResponseMessage());
83+
return connection.getInputStream();
84+
}
85+
86+
private static boolean isHttpProtocol(String protocol) {
87+
return "http".equalsIgnoreCase(protocol) || "https".equalsIgnoreCase(protocol);
88+
}
89+
90+
private static String removeUserInfo(URI uri) throws URISyntaxException {
91+
var rawAuthority = uri.getRawAuthority();
92+
if ( rawAuthority!=null ) {
93+
var atSignIndex = rawAuthority.lastIndexOf('@');
94+
if ( atSignIndex >= 0 ) {
95+
rawAuthority = rawAuthority.substring(atSignIndex + 1);
96+
}
97+
}
98+
return new URI(
99+
uri.getScheme(),
100+
rawAuthority,
101+
uri.getRawPath(),
102+
uri.getRawQuery(),
103+
uri.getRawFragment()
104+
).toString();
105+
}
106+
107+
private static Map<String, String> parseHeaders(String rawUserInfo) {
108+
if ( StringUtils.isBlank(rawUserInfo) ) { return Collections.emptyMap(); }
109+
if ( rawUserInfo.startsWith(PREFIX_BEARER) ) {
110+
return Map.of("Authorization", "Bearer "+decode(rawUserInfo.substring(PREFIX_BEARER.length())));
111+
}
112+
if ( rawUserInfo.startsWith(PREFIX_HEADER) ) {
113+
return parseHeaderAssignments(rawUserInfo.substring(PREFIX_HEADER.length()));
114+
}
115+
if ( rawUserInfo.startsWith(PREFIX_HEADERS) ) {
116+
return parseHeaderAssignments(rawUserInfo.substring(PREFIX_HEADERS.length()));
117+
}
118+
return parseBasicAuth(rawUserInfo);
119+
}
120+
121+
private static Map<String, String> parseBasicAuth(String rawUserInfo) {
122+
var separatorIndex = rawUserInfo.indexOf(':');
123+
String username;
124+
String password;
125+
if ( separatorIndex < 0 ) {
126+
username = decode(rawUserInfo);
127+
password = "";
128+
} else {
129+
username = decode(rawUserInfo.substring(0, separatorIndex));
130+
password = decode(rawUserInfo.substring(separatorIndex + 1));
131+
}
132+
var value = Base64.getEncoder().encodeToString((username+":"+password).getBytes(StandardCharsets.UTF_8));
133+
return Map.of("Authorization", "Basic "+value);
134+
}
135+
136+
private static Map<String, String> parseHeaderAssignments(String assignments) {
137+
if ( StringUtils.isBlank(assignments) ) {
138+
throw new IllegalArgumentException("No headers specified in URL userinfo");
139+
}
140+
var result = new LinkedHashMap<String, String>();
141+
for ( var assignment : assignments.split("&") ) {
142+
if ( StringUtils.isBlank(assignment) ) { continue; }
143+
var separatorIndex = assignment.indexOf('=');
144+
if ( separatorIndex <= 0 ) {
145+
throw new IllegalArgumentException("Invalid header assignment in URL userinfo: "+assignment);
146+
}
147+
var name = decode(assignment.substring(0, separatorIndex));
148+
var value = decode(assignment.substring(separatorIndex + 1));
149+
if ( StringUtils.isBlank(name) ) {
150+
throw new IllegalArgumentException("Header name must not be blank in URL userinfo");
151+
}
152+
result.put(name, value);
153+
}
154+
if ( result.isEmpty() ) {
155+
throw new IllegalArgumentException("No valid headers specified in URL userinfo");
156+
}
157+
return Collections.unmodifiableMap(result);
158+
}
159+
160+
/**
161+
* URLDecoder treats '+' as a space; preserve literal '+' characters in userinfo.
162+
*/
163+
private static String decode(String value) {
164+
return URLDecoder.decode(value.replace("+", "%2B"), StandardCharsets.UTF_8);
165+
}
166+
167+
public static final class ParsedRemoteUrl {
168+
private final String requestUrl;
169+
private final Map<String, String> headers;
170+
171+
private ParsedRemoteUrl(String requestUrl, Map<String, String> headers) {
172+
this.requestUrl = requestUrl;
173+
this.headers = headers;
174+
}
175+
176+
public String getRequestUrl() {
177+
return requestUrl;
178+
}
179+
180+
public Map<String, String> getHeaders() {
181+
return headers;
182+
}
183+
}
184+
}

‎fcli-core/fcli-common-core/src/main/java/com/fortify/cli/common/rest/unirest/UnirestHelper.java‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,24 @@
2828
*/
2929
public class UnirestHelper {
3030
public static final File download(String fcliModule, String url, File dest) {
31+
var parsedUrl = parseRemoteUrl(url);
3132
try (var unirest = createUnirestInstance()) {
32-
ProxyHelper.configureProxy(unirest, fcliModule, url);
33-
unirest.get(url).asFile(dest.getAbsolutePath(), StandardCopyOption.REPLACE_EXISTING).getBody();
33+
ProxyHelper.configureProxy(unirest, fcliModule, parsedUrl.getRequestUrl());
34+
var request = unirest.get(parsedUrl.getRequestUrl());
35+
parsedUrl.getHeaders().forEach(request::headerReplace);
36+
request.asFile(dest.getAbsolutePath(), StandardCopyOption.REPLACE_EXISTING).getBody();
3437
return dest;
3538
}
3639
}
3740

41+
private static RemoteUrlAuthHelper.ParsedRemoteUrl parseRemoteUrl(String url) {
42+
try {
43+
return RemoteUrlAuthHelper.parse(url);
44+
} catch (Exception e) {
45+
throw new IllegalArgumentException("Invalid URL: "+url, e);
46+
}
47+
}
48+
3849
/**
3950
* Create a new Unirest instance, configured with the standard FCLI JSON object mapper.
4051
* Callers are responsible for closing the returned instance.

0 commit comments

Comments
 (0)