Skip to content

Commit 34a97f8

Browse files
bhermannbhermann
authored
Merge pull request #38 from delphi-hub/bug/34/ResultsSizeError
Bug/34/results size error
2 parents 8ce9788 + 8760666 commit 34a97f8

File tree

7 files changed

+192
-21
lines changed

7 files changed

+192
-21
lines changed

‎src/it/scala/de/upb/cs/swt/delphi/webapi/SearchQueryTest.scala

+13-6
Original file line numberDiff line numberDiff line change
@@ -16,17 +16,24 @@
1616

1717
package de.upb.cs.swt.delphi.webapi
1818

19-
import de.upb.cs.swt.delphi.webapi.search.{QueryRequest, SearchQuery}
19+
import de.upb.cs.swt.delphi.webapi.search.{QueryRequest, SearchError, SearchQuery}
2020
import org.scalatest.{FlatSpec, Matchers}
2121

22-
import scala.util.Success
22+
import scala.util.Failure
2323

2424
class SearchQueryTest extends FlatSpec with Matchers {
25-
"Search query" should "check for fields" in {
25+
"Search query" should "fail on large request limit" in {
2626
val configuration = new Configuration()
2727
val q = new SearchQuery(configuration, new FeatureQuery(configuration))
28-
29-
val response = q.search(QueryRequest("[if_icmpeq (opcode:159)]>1"))
30-
response shouldBe a [Success[_]]
28+
val size = 20000
29+
val response = q.search(QueryRequest("[dstore_1 (opcode:72)]<1", Some(size)))
30+
response match {
31+
case Failure(exception) => {
32+
exception shouldBe a[SearchError]
33+
}
34+
case _ => {
35+
fail("Limit exceeded should fail")
36+
}
37+
}
3138
}
3239
}

‎src/main/scala/de/upb/cs/swt/delphi/webapi/DelphiRoutes.scala

+11-2
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ import de.upb.cs.swt.delphi.webapi.IpLogActor._
2727
import de.upb.cs.swt.delphi.webapi.StatisticsJson._
2828
import de.upb.cs.swt.delphi.webapi.artifacts.ArtifactJson._
2929
import de.upb.cs.swt.delphi.webapi.search.QueryRequestJson._
30-
import de.upb.cs.swt.delphi.webapi.search.{QueryRequest, SearchQuery}
30+
import de.upb.cs.swt.delphi.webapi.search.{QueryRequest, SearchError, SearchQuery}
3131
import spray.json._
3232

3333
import scala.concurrent.duration._
@@ -118,7 +118,16 @@ class DelphiRoutes(requestLimiter: RequestLimitScheduler) extends JsonSupport wi
118118
complete(
119119
new SearchQuery(configuration, featureExtractor).search(input) match {
120120
case Success(result) => prettyPrint(pretty, result.toJson)
121-
case Failure(e) => e.getMessage
121+
case Failure(e) => {
122+
e match {
123+
case se: SearchError => {
124+
se.toJson
125+
}
126+
case _ => {
127+
new SearchError("Search query failed").toJson
128+
}
129+
}
130+
}
122131
}
123132
)
124133
}

‎src/main/scala/de/upb/cs/swt/delphi/webapi/search/QueryRequest.scala

+16-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,18 @@
1+
// Copyright (C) 2018 The Delphi Team.
2+
// See the LICENCE file distributed with this work for additional
3+
// information regarding copyright ownership.
4+
//
5+
// Licensed under the Apache License, Version 2.0 (the "License");
6+
// you may not use this file except in compliance with the License.
7+
// You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing, software
12+
// distributed under the License is distributed on an "AS IS" BASIS,
13+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
// See the License for the specific language governing permissions and
15+
// limitations under the License.
116
package de.upb.cs.swt.delphi.webapi.search
217

3-
case class QueryRequest (query : String, limit : Option[Int] = Some(50))
18+
case class QueryRequest (query : String, limit : Option[Int] = Some(defaultFetchSize))

‎src/main/scala/de/upb/cs/swt/delphi/webapi/search/QueryRequestJson.scala

+15
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,18 @@
1+
// Copyright (C) 2018 The Delphi Team.
2+
// See the LICENCE file distributed with this work for additional
3+
// information regarding copyright ownership.
4+
//
5+
// Licensed under the Apache License, Version 2.0 (the "License");
6+
// you may not use this file except in compliance with the License.
7+
// You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing, software
12+
// distributed under the License is distributed on an "AS IS" BASIS,
13+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
// See the License for the specific language governing permissions and
15+
// limitations under the License.
116
package de.upb.cs.swt.delphi.webapi.search
217

318
import spray.json.DefaultJsonProtocol

‎src/main/scala/de/upb/cs/swt/delphi/webapi/search/SearchQuery.scala

+38-12
Original file line numberDiff line numberDiff line change
@@ -18,18 +18,18 @@ package de.upb.cs.swt.delphi.webapi.search
1818

1919
import com.sksamuel.elastic4s.http.ElasticDsl._
2020
import com.sksamuel.elastic4s.http.search.SearchHits
21-
import com.sksamuel.elastic4s.http.{ElasticClient, RequestSuccess}
21+
import com.sksamuel.elastic4s.http.{ElasticClient, RequestFailure, RequestSuccess}
2222
import com.sksamuel.elastic4s.searches.queries.{NoopQuery, Query}
23-
import de.upb.cs.swt.delphi.webapi.{Configuration, FeatureQuery}
24-
import de.upb.cs.swt.delphi.webapi.artifacts.ArtifactTransformer
23+
import de.upb.cs.swt.delphi.webapi.artifacts.{Artifact, ArtifactTransformer}
2524
import de.upb.cs.swt.delphi.webapi.search.querylanguage._
25+
import de.upb.cs.swt.delphi.webapi.{Configuration, FeatureQuery}
2626

2727
import scala.util.{Failure, Success, Try}
2828

2929
class SearchQuery(configuration: Configuration, featureExtractor: FeatureQuery) {
3030
private val client = ElasticClient(configuration.elasticsearchClientUri)
3131

32-
private def checkAndExecuteParsedQuery(ast: CombinatorialExpr, limit : Int): Try[SearchHits] = {
32+
private def checkAndExecuteParsedQuery(ast: CombinatorialExpr, limit: Int): Try[SearchHits] = {
3333
val fields = collectFieldNames(ast)
3434
if (fields.diff(featureExtractor.featureList.toSeq).size > 0) return Failure(new IllegalArgumentException("Unknown field name used."))
3535

@@ -109,16 +109,42 @@ class SearchQuery(configuration: Configuration, featureExtractor: FeatureQuery)
109109
}
110110
}
111111

112-
def search(query: QueryRequest) = {
113-
val parserResult = new Syntax(query.query).QueryRule.run()
114-
parserResult match {
115-
case Failure(e) => Failure(e)
116-
case Success(ast) => {
117-
checkAndExecuteParsedQuery(ast, query.limit.getOrElse(50)) match {
118-
case Failure(e) => Failure(e)
119-
case Success(hits) => Success(ArtifactTransformer.transformResults(hits))
112+
def checkValidSize: Option[Int] = {
113+
import elastic4s.extns._
114+
import elastic4s.extns.ElasticDslExtn._
115+
val params = Map("include_defaults" -> true)
116+
val query = SettingsRequest("delphi", params)
117+
val res = client.execute {
118+
query
119+
}.await
120+
res match {
121+
case RequestSuccess(_, b, _, _) => {
122+
maxResultSize(b, configuration)
123+
}
124+
case RequestFailure(_, _, _, _) => {
125+
None
126+
}
127+
}
128+
}
129+
130+
def search(query: QueryRequest): Try[Array[Artifact]] = {
131+
lazy val size = checkValidSize
132+
val validSize = size.exists(query.limit.getOrElse(defaultFetchSize) <= _)
133+
if (validSize) {
134+
val parserResult = new Syntax(query.query).QueryRule.run()
135+
parserResult match {
136+
case Failure(e) => Failure(e)
137+
case Success(ast) => {
138+
checkAndExecuteParsedQuery(ast, query.limit.getOrElse(defaultFetchSize)) match {
139+
case Failure(e) => Failure(e)
140+
case Success(hits) => Success(ArtifactTransformer.transformResults(hits))
141+
}
120142
}
121143
}
122144
}
145+
else {
146+
val errorMsg = new SearchError(s"Query limit exceeded default limit: ${query.limit}>${size}")
147+
Failure(errorMsg)
148+
}
123149
}
124150
}

‎src/main/scala/de/upb/cs/swt/delphi/webapi/search/elastic4s/extns/package.scala

+66
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
// Copyright (C) 2018 The Delphi Team.
2+
// See the LICENCE file distributed with this work for additional
3+
// information regarding copyright ownership.
4+
//
5+
// Licensed under the Apache License, Version 2.0 (the "License");
6+
// you may not use this file except in compliance with the License.
7+
// You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing, software
12+
// distributed under the License is distributed on an "AS IS" BASIS,
13+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
// See the License for the specific language governing permissions and
15+
// limitations under the License.
16+
package de.upb.cs.swt.delphi.webapi.search.elastic4s
17+
18+
import com.sksamuel.elastic4s.http.settings.IndexSettingsResponse
19+
import com.sksamuel.elastic4s.http.{ElasticRequest, Handler}
20+
import com.sksamuel.elastic4s.json.JacksonSupport
21+
import de.upb.cs.swt.delphi.webapi.Configuration
22+
23+
package object extns {
24+
25+
case class SettingsRequest(index: String, params: Map[String, Any])
26+
27+
trait RichSettingsHandler {
28+
29+
implicit object RichGetSettings extends Handler[SettingsRequest, IndexSettingsResponse] {
30+
31+
override def build(request: SettingsRequest): ElasticRequest = {
32+
val endpoint = "/" + request.index + "/_settings"
33+
val req = ElasticRequest("GET", endpoint, request.params)
34+
req
35+
}
36+
}
37+
38+
}
39+
40+
def maxResultSize(res: Option[String], config: Configuration): Option[Int] = {
41+
res match {
42+
case Some(j) => {
43+
val custom = s"/${config.esIndex}/settings/index"
44+
val default = s"/${config.esIndex}/defaults/index"
45+
val target = "max_result_window"
46+
val node = JacksonSupport.mapper.readTree(j)
47+
val size = if (node.at(custom).has(target)) {
48+
Some(node.at(custom + "/" + target).asInt())
49+
}
50+
else {
51+
Some(node.at(default + "/" + target).asInt())
52+
}
53+
size
54+
}
55+
case None => {
56+
None
57+
}
58+
}
59+
}
60+
61+
62+
trait ElasticDslExtn extends RichSettingsHandler
63+
64+
object ElasticDslExtn extends ElasticDslExtn
65+
66+
}

‎src/main/scala/de/upb/cs/swt/delphi/webapi/search/package.scala

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
// Copyright (C) 2018 The Delphi Team.
2+
// See the LICENCE file distributed with this work for additional
3+
// information regarding copyright ownership.
4+
//
5+
// Licensed under the Apache License, Version 2.0 (the "License");
6+
// you may not use this file except in compliance with the License.
7+
// You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing, software
12+
// distributed under the License is distributed on an "AS IS" BASIS,
13+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
// See the License for the specific language governing permissions and
15+
// limitations under the License.
16+
17+
package de.upb.cs.swt.delphi.webapi
18+
19+
import spray.json._
20+
21+
package object search {
22+
23+
val defaultFetchSize = 50
24+
25+
class SearchError(msg: String) extends RuntimeException(msg) with JsonSupport
26+
27+
implicit val searchErrorWriter = new JsonWriter[SearchError] {
28+
override def write(obj: SearchError): JsValue = {
29+
JsObject("msg" -> JsString(obj.getMessage))
30+
}
31+
}
32+
33+
}

0 commit comments

Comments
 (0)