Skip to content

Commit 945272a

Browse files
ruhan1ruhan1
authored
Use base64 to take query string into encoded path (#61)
* Use base64 to take query string into encoded path * Disable sonar:sonar in Jenkinsfile * Set path-encode attribute for proxied repo
1 parent ed67e34 commit 945272a

File tree

11 files changed

+136
-39
lines changed

11 files changed

+136
-39
lines changed

‎Jenkinsfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ pipeline {
1919
expression { env.CHANGE_ID != null } // Pull request
2020
}
2121
steps {
22-
sh '${M2_HOME}/bin/mvn -Dplugin.jacoco.skip=false -B -V clean verify sonar:sonar -Prun-its'
22+
sh '${M2_HOME}/bin/mvn -Dplugin.jacoco.skip=false -B -V clean verify -Prun-its'
2323
}
2424
}
2525
stage('Load OCP Mappings') {

‎src/main/java/org/commonjava/indy/service/httprox/handler/AbstractProxyRepositoryCreator.java

+10
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,17 @@
2828
import static org.commonjava.indy.model.core.ArtifactStore.TRACKING_ID;
2929
import static org.commonjava.indy.model.core.GenericPackageTypeDescriptor.GENERIC_PKG_KEY;
3030
import static org.commonjava.indy.model.core.PathStyle.hashed;
31+
import static org.commonjava.indy.model.core.StoreType.group;
3132
import static org.commonjava.indy.service.httprox.util.HttpProxyConstants.PROXY_REPO_PREFIX;
33+
import static org.commonjava.indy.service.httprox.util.UrlUtils.hasQueryParam;
3234

3335
public abstract class AbstractProxyRepositoryCreator
3436
implements ProxyRepositoryCreator
3537
{
38+
private static final String ATTR_PATH_ENCODE = "path-encode";
39+
40+
private static final String PATH_ENCODE_BASE64 = "base64";
41+
3642
@Override
3743
public abstract ProxyCreationResult create(String trackingID, String name, String baseUrl, UrlInfo urlInfo,
3844
UserPass userPass, Logger logger );
@@ -116,6 +122,10 @@ private void setPropsAndMetadata(ArtifactStore store, String trackingID, UrlInfo
116122
{
117123
store.setMetadata( TRACKING_ID, trackingID );
118124
}
125+
if ( store.getType() != group && hasQueryParam(info.getUrl()) )
126+
{
127+
store.setMetadata( ATTR_PATH_ENCODE, PATH_ENCODE_BASE64 );
128+
}
119129
}
120130

121131
/**

‎src/main/java/org/commonjava/indy/service/httprox/handler/ProxyMITMSSLServer.java

+6-9
Original file line numberDiff line numberDiff line change
@@ -211,7 +211,7 @@ else if ( line.isEmpty() )
211211
}
212212
}
213213

214-
logger.debug( "Request:\n{}", sb.toString() );
214+
logger.debug( "Request:\n{}", sb );
215215

216216
if ( path != null )
217217
{
@@ -268,21 +268,18 @@ else if ( line.isEmpty() )
268268
}
269269
}
270270

271-
private void transferRemote( Socket socket, String host, int port, String method, String path, ProxyMeter meter ) throws Exception
271+
private void transferRemote( Socket socket, String host, int port, String method, String file,
272+
ProxyMeter meter ) throws Exception
272273
{
273274
String protocol = "https";
274-
String auth = null;
275-
String query = null;
276-
String fragment = null;
277-
URI uri = new URI( protocol, auth, host, port, path, query, fragment );
278-
URL remoteUrl = uri.toURL();
279-
logger.debug( "Requesting remote URL: {}", remoteUrl.toString() );
275+
URL remoteUrl = new URL( protocol, host, port, file );
276+
logger.debug( "Requesting remote URL: {}", remoteUrl );
280277

281278
ArtifactStore store = proxyResponseHelper.getArtifactStore( trackingId, remoteUrl );
282279
try (BufferedOutputStream out = new BufferedOutputStream( socket.getOutputStream() );
283280
HttpConduitWrapper http = new HttpConduitWrapper( new OutputStreamSinkChannel( out ), null ))
284281
{
285-
proxyResponseHelper.transfer( http, store, remoteUrl.getPath(), GET_METHOD.equals( method ),
282+
proxyResponseHelper.transfer( http, store, remoteUrl.getFile(), GET_METHOD.equals( method ),
286283
proxyUserPass, meter );
287284
out.flush();
288285
}

‎src/main/java/org/commonjava/indy/service/httprox/handler/ProxyResponseWriter.java

+4-2
Original file line numberDiff line numberDiff line change
@@ -227,9 +227,11 @@ private void doHandleEvent(final ConduitStreamSinkChannel sinkChannel)
227227
case HEAD_METHOD:
228228
{
229229
final URL url = new URL( requestLine.getUri() );
230-
logger.debug( "getArtifactStore starts, trackingId: {}, url: {}", trackingId, url );
230+
logger.debug( "Get artifact store, trackingId: {}, url: {}", trackingId, url );
231231
ArtifactStore store = proxyResponseHelper.getArtifactStore( trackingId, url );
232-
proxyResponseHelper.transfer( http, store, url.getPath(), GET_METHOD.equals( method ), proxyUserPass, meter );
232+
// 'url.getFile()' gets the file name of this URL. The returned file portion will be the
233+
// same as getPath(), plus the concatenation of the value of getQuery(), if any.
234+
proxyResponseHelper.transfer( http, store, url.getFile(), GET_METHOD.equals( method ), proxyUserPass, meter );
233235
break;
234236
}
235237
case OPTIONS_METHOD:

‎src/main/java/org/commonjava/indy/service/httprox/util/ProxyResponseHelper.java

+13-9
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import io.vertx.core.http.HttpMethod;
2121
import kotlin.Pair;
2222
import okhttp3.ResponseBody;
23+
import org.apache.commons.codec.binary.Base64;
2324
import org.apache.commons.lang3.StringUtils;
2425
import org.apache.http.HttpRequest;
2526
import org.apache.http.HttpStatus;
@@ -41,7 +42,9 @@
4142
import javax.ws.rs.core.Response;
4243
import java.io.IOException;
4344
import java.io.InputStream;
45+
import java.io.UnsupportedEncodingException;
4446
import java.net.URL;
47+
import java.time.Duration;
4548
import java.util.List;
4649
import java.util.concurrent.TimeUnit;
4750
import java.util.function.Predicate;
@@ -55,6 +58,7 @@
5558
import static org.commonjava.indy.service.httprox.util.MetricsConstants.CONTENT_ENTRY_POINT;
5659
import static org.commonjava.indy.service.httprox.util.MetricsConstants.PATH;
5760
import static org.commonjava.indy.service.httprox.util.MetricsConstants.METADATA_CONTENT;
61+
import static org.commonjava.indy.service.httprox.util.UrlUtils.base64url;
5862

5963
public class ProxyResponseHelper
6064
{
@@ -66,7 +70,7 @@ public class ProxyResponseHelper
6670

6771
private final ProxyConfiguration config;
6872

69-
private boolean transferred;
73+
private volatile boolean transferred;
7074

7175
private ProxyRepositoryCreator repoCreator;
7276

@@ -389,10 +393,10 @@ private void doTransfer( final HttpConduitWrapper http, final ArtifactStore stor
389393
{
390394
if ( transferred )
391395
{
396+
logger.info("Transfer already done, store: {}, path: {}", store.getKey(), path);
392397
return;
393398
}
394399

395-
transferred = true;
396400
if ( !http.isOpen() )
397401
{
398402
throw new IOException( "Sink channel already closed (or null)!" );
@@ -412,7 +416,10 @@ private void doTransfer( final HttpConduitWrapper http, final ArtifactStore stor
412416
}
413417

414418
try {
415-
Uni<okhttp3.Response> responseUni = contentRetrievalService.doGet(trackingId, store.getType().name(), store.getName(), path);
419+
String encodedPath = base64url(path);
420+
logger.debug( "Get from content service, store: {}, path: {}", store.getKey(), encodedPath );
421+
Uni<okhttp3.Response> responseUni = contentRetrievalService.doGet(trackingId, store.getType().name(),
422+
store.getName(), encodedPath);
416423

417424
responseUni.subscribe().with(
418425
response ->
@@ -438,7 +445,7 @@ private void doTransfer( final HttpConduitWrapper http, final ArtifactStore stor
438445
}
439446
finally
440447
{
441-
transferred = false;
448+
transferred = true;
442449
if ( response != null && responseBody != null )
443450
{
444451
responseBody.close();
@@ -457,15 +464,12 @@ private void doTransfer( final HttpConduitWrapper http, final ArtifactStore stor
457464
}
458465
finally
459466
{
460-
transferred = false;
467+
transferred = true;
461468
}
462469
}
463470
);
464471

465-
while ( transferred )
466-
{
467-
TimeUnit.MILLISECONDS.sleep( 100 );
468-
}
472+
responseUni.await().atMost(Duration.ofMinutes(5)); // Wait until the item or a failure is emitted
469473

470474
if ( meter != null )
471475
{

‎src/main/java/org/commonjava/indy/service/httprox/util/UrlUtils.java

+20
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,12 @@
1515
*/
1616
package org.commonjava.indy.service.httprox.util;
1717

18+
import org.apache.commons.codec.binary.Base64;
1819
import org.slf4j.Logger;
1920
import org.slf4j.LoggerFactory;
2021

22+
import java.io.UnsupportedEncodingException;
23+
import java.nio.charset.StandardCharsets;
2124
import java.util.ArrayList;
2225
import java.util.List;
2326
import java.util.Map;
@@ -165,4 +168,21 @@ public static String normalizePath( final String... path )
165168

166169
return sb.toString();
167170
}
171+
172+
/**
173+
* Encode path using a URL-safe base64 algorithm if there are query parameters.
174+
*/
175+
public static String base64url(String path)
176+
{
177+
if ( hasQueryParam(path) )
178+
{
179+
return Base64.encodeBase64URLSafeString(path.getBytes(StandardCharsets.UTF_8));
180+
}
181+
return path;
182+
}
183+
184+
public static boolean hasQueryParam( String path )
185+
{
186+
return path.contains("?");
187+
}
168188
}

‎src/main/resources/application.yaml

+5
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
11
quarkus:
22
log:
33
level: INFO
4+
category:
5+
"org.commonjava.indy.service.httprox":
6+
level: DEBUG
7+
"org.commonjava.indy.model":
8+
level: ERROR
49
http:
510
host-enabled: false
611
opentelemetry:

‎src/test/java/org/commonjava/service/httprox/AbstractGenericProxyTest.java

+12-3
Original file line numberDiff line numberDiff line change
@@ -41,12 +41,13 @@
4141
import org.mockito.Mockito;
4242

4343
import java.io.*;
44+
import java.nio.charset.StandardCharsets;
4445
import java.security.KeyStore;
4546

47+
import static org.commonjava.indy.service.httprox.util.UrlUtils.base64url;
4648
import static org.hamcrest.CoreMatchers.notNullValue;
4749
import static org.hamcrest.MatcherAssert.assertThat;
48-
import static org.mockito.ArgumentMatchers.any;
49-
import static org.mockito.ArgumentMatchers.contains;
50+
import static org.mockito.ArgumentMatchers.*;
5051

5152
public class AbstractGenericProxyTest
5253
{
@@ -65,7 +66,8 @@ public static void setup() throws Exception
6566

6667
Mockito.when(contentRetrievalService.doGet(any(), any(), any(), contains("indy-api-1.3.1.pom"))).thenReturn(Uni.createFrom().item(buildResponse("indy-api-1.3.1.pom")));
6768
Mockito.when(contentRetrievalService.doGet(any(), any(), any(), contains("fsevents-1.2.4.tgz"))).thenReturn(Uni.createFrom().item(buildResponse("fsevents-1.2.4.tgz")));
68-
Mockito.when(contentRetrievalService.doGet(any(), any(), any(), contains("simple.pom"))).thenReturn(Uni.createFrom().item(buildResponse("simple.pom")));
69+
Mockito.when(contentRetrievalService.doGet(any(), any(), any(), eq("/test/org/test/simple/1/simple.pom"))).thenReturn(Uni.createFrom().item(buildResponse("simple.pom")));
70+
Mockito.when(contentRetrievalService.doGet(any(), any(), any(), eq(base64url("/org/test/simple.pom?version=2.0")))).thenReturn(Uni.createFrom().item(buildResponse("simple-2.0.pom")));
6971
Mockito.when(contentRetrievalService.doGet(any(), any(), any(), contains("simple-1.pom"))).thenReturn(Uni.createFrom().item(buildResponse("simple-1.pom")));
7072
Mockito.when(contentRetrievalService.doGet(any(), any(), any(), contains("no.pom"))).thenReturn(Uni.createFrom().item(buildResponse("no.pom")));
7173

@@ -227,4 +229,11 @@ protected static void copyToConfigFile( String resourcePath, String path ) throw
227229
Thread.currentThread().getContextClassLoader().getResourceAsStream( resourcePath ), file );
228230
}
229231

232+
protected String loadResource(String resource) throws IOException
233+
{
234+
final InputStream stream = Thread.currentThread().getContextClassLoader().getResourceAsStream( resource );
235+
assert stream != null;
236+
return IOUtils.toString( stream, StandardCharsets.UTF_8);
237+
}
238+
230239
}

‎src/test/java/org/commonjava/service/httprox/HttpProxyTest.java

+28-13
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,8 @@
3636

3737
import javax.inject.Inject;
3838
import javax.ws.rs.WebApplicationException;
39-
import java.io.IOException;
4039
import java.io.InputStream;
40+
import java.nio.charset.StandardCharsets;
4141

4242
import static org.hamcrest.CoreMatchers.equalTo;
4343
import static org.hamcrest.CoreMatchers.notNullValue;
@@ -63,8 +63,6 @@ public class HttpProxyTest extends AbstractGenericProxyTest
6363
public void proxySimplePomAndAutoCreateRemoteRepo()
6464
throws Exception
6565
{
66-
final String testRepo = "test";
67-
6866
final String url = "http://remote.example:80/test/org/test/simple/1/simple.pom";
6967
final String testPomContents = loadResource("simple.pom");
7068

@@ -86,10 +84,35 @@ public void proxySimplePomAndAutoCreateRemoteRepo()
8684
{
8785
IOUtils.closeQuietly( stream );
8886
}
87+
}
88+
89+
/**
90+
* If path contains '?', the proxy will create the remote repo with attribute 'path-encode':'base64'. When sending
91+
* the request to remote site, the 'path+query' will be encoded.
92+
*/
93+
@Test
94+
public void proxySimplePomWithQueryParameter()
95+
throws Exception
96+
{
97+
final String url = "http://remote.example:80/org/test/simple.pom?version=2.0";
98+
final String testPomContents = loadResource("simple-2.0.pom");
8999

90-
//TODO check if remote repo created
91-
//repositoryService.repoExists("GENERIC_PKG_KEY", StoreType.remote.name(), "httprox_remote-example_80");
100+
final HttpGet get = new HttpGet( url );
101+
final CloseableHttpClient client = proxiedHttp();
102+
InputStream stream = null;
103+
try
104+
{
105+
CloseableHttpResponse response = client.execute( get, proxyContext( USER, PASS ) );
106+
stream = response.getEntity().getContent();
107+
final String resultingPom = IOUtils.toString( stream, StandardCharsets.UTF_8);
92108

109+
assertThat( resultingPom, notNullValue() );
110+
assertThat( resultingPom, equalTo( testPomContents ) );
111+
}
112+
finally
113+
{
114+
IOUtils.closeQuietly( stream );
115+
}
93116
}
94117

95118
@Test
@@ -138,12 +161,4 @@ protected CloseableHttpClient proxiedHttp()
138161
return HttpClients.custom().setRoutePlanner( planner ).build();
139162
}
140163

141-
protected String loadResource(String resource) throws IOException
142-
{
143-
final InputStream stream = Thread.currentThread().getContextClassLoader().getResourceAsStream( resource );
144-
145-
return IOUtils.toString( stream );
146-
}
147-
148-
149164
}

‎src/test/java/org/commonjava/service/httprox/ProxyHttpsTest.java

+14-2
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919
import org.junit.jupiter.api.Test;
2020

2121
import static io.smallrye.common.constraint.Assert.assertTrue;
22+
import static org.hamcrest.CoreMatchers.equalTo;
23+
import static org.hamcrest.MatcherAssert.assertThat;
2224

2325
@QuarkusTest
2426
public class ProxyHttpsTest extends AbstractGenericProxyTest
@@ -28,14 +30,24 @@ public class ProxyHttpsTest extends AbstractGenericProxyTest
2830

2931
private static final String PASS = "password";
3032

31-
String https_url =
33+
final String httpsUrl =
3234
"https://oss.sonatype.org/content/repositories/releases/org/commonjava/indy/indy-api/1.3.1/indy-api-1.3.1.pom";
3335

3436
@Test
3537
public void run() throws Exception
3638
{
37-
String ret = get( https_url, true, USER, PASS );
39+
String ret = get( httpsUrl, true, USER, PASS );
3840
assertTrue( ret.contains( "<artifactId>indy-api</artifactId>" ) );
3941

4042
}
43+
44+
final String httpsUrlWithQuery = "https://really.useful.script/org/test/simple.pom?version=2.0";
45+
46+
@Test
47+
public void runWithQuery() throws Exception
48+
{
49+
String ret = get( httpsUrlWithQuery, true, USER, PASS );
50+
final String expected = loadResource("simple-2.0.pom");
51+
assertThat( ret, equalTo( expected ) );
52+
}
4153
}

‎src/test/resources/simple-2.0.pom

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
<!--
2+
Copyright (C) 2011-2018 Red Hat, Inc. (https://github.com/Commonjava/indy)
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
16+
-->
17+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
18+
<modelVersion>4.0.0</modelVersion>
19+
20+
<groupId>org.test</groupId>
21+
<artifactId>simple</artifactId>
22+
<version>2.0</version>
23+
</project>

0 commit comments

Comments
 (0)