Skip to content

Commit 67f6151

Browse files
jonhealy1jonhealy1
authored
Merge pull request stac-utils#204 from stac-utils/revert-203-update-4-mongo
Revert "Update makefile etc "
2 parents a6a8634 + 8041a0f commit 67f6151

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

59 files changed

+4937
-1315
lines changed

‎.github/workflows/cicd.yml

+108-107
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
name: stac-fastapi-mongo
1+
name: stac-fastapi-elasticsearch
2+
on:
23
push:
34
branches:
45
- main
@@ -7,123 +8,123 @@ name: stac-fastapi-mongo
78
- main
89
- features/**
910

10-
# jobs:
11-
# test:
12-
# runs-on: ubuntu-latest
13-
# timeout-minutes: 10
11+
jobs:
12+
test:
13+
runs-on: ubuntu-latest
14+
timeout-minutes: 10
1415

15-
# services:
16+
services:
1617

17-
# elasticsearch_8_svc:
18-
# image: docker.elastic.co/elasticsearch/elasticsearch:8.11.0
19-
# env:
20-
# cluster.name: stac-cluster
21-
# node.name: es01
22-
# network.host: 0.0.0.0
23-
# transport.host: 0.0.0.0
24-
# discovery.type: single-node
25-
# http.port: 9200
26-
# xpack.license.self_generated.type: basic
27-
# xpack.security.enabled: false
28-
# xpack.security.transport.ssl.enabled: false
29-
# ES_JAVA_OPTS: -Xms512m -Xmx1g
30-
# ports:
31-
# - 9200:9200
18+
elasticsearch_8_svc:
19+
image: docker.elastic.co/elasticsearch/elasticsearch:8.11.0
20+
env:
21+
cluster.name: stac-cluster
22+
node.name: es01
23+
network.host: 0.0.0.0
24+
transport.host: 0.0.0.0
25+
discovery.type: single-node
26+
http.port: 9200
27+
xpack.license.self_generated.type: basic
28+
xpack.security.enabled: false
29+
xpack.security.transport.ssl.enabled: false
30+
ES_JAVA_OPTS: -Xms512m -Xmx1g
31+
ports:
32+
- 9200:9200
3233

33-
# elasticsearch_7_svc:
34-
# image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
35-
# env:
36-
# cluster.name: stac-cluster
37-
# node.name: es01
38-
# network.host: 0.0.0.0
39-
# transport.host: 0.0.0.0
40-
# discovery.type: single-node
41-
# http.port: 9400
42-
# xpack.license.self_generated.type: basic
43-
# xpack.security.enabled: false
44-
# xpack.security.transport.ssl.enabled: false
45-
# ES_JAVA_OPTS: -Xms512m -Xmx1g
46-
# ports:
47-
# - 9400:9400
34+
elasticsearch_7_svc:
35+
image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
36+
env:
37+
cluster.name: stac-cluster
38+
node.name: es01
39+
network.host: 0.0.0.0
40+
transport.host: 0.0.0.0
41+
discovery.type: single-node
42+
http.port: 9400
43+
xpack.license.self_generated.type: basic
44+
xpack.security.enabled: false
45+
xpack.security.transport.ssl.enabled: false
46+
ES_JAVA_OPTS: -Xms512m -Xmx1g
47+
ports:
48+
- 9400:9400
4849

49-
# opensearch_2_11:
50-
# image: opensearchproject/opensearch:2.11.1
51-
# env:
52-
# cluster.name: stac-cluster
53-
# node.name: os01
54-
# network.host: 0.0.0.0
55-
# transport.host: 0.0.0.0
56-
# discovery.type: single-node
57-
# http.port: 9202
58-
# http.cors.enabled: true
59-
# plugins.security.disabled: true
60-
# plugins.security.ssl.http.enabled: true
61-
# OPENSEARCH_JAVA_OPTS: -Xms512m -Xmx512m
62-
# ports:
63-
# - 9202:9202
64-
# strategy:
65-
# matrix:
66-
# python-version: [ "3.8", "3.9", "3.10", "3.11"]
50+
opensearch_2_11:
51+
image: opensearchproject/opensearch:2.11.1
52+
env:
53+
cluster.name: stac-cluster
54+
node.name: os01
55+
network.host: 0.0.0.0
56+
transport.host: 0.0.0.0
57+
discovery.type: single-node
58+
http.port: 9202
59+
http.cors.enabled: true
60+
plugins.security.disabled: true
61+
plugins.security.ssl.http.enabled: true
62+
OPENSEARCH_JAVA_OPTS: -Xms512m -Xmx512m
63+
ports:
64+
- 9202:9202
65+
strategy:
66+
matrix:
67+
python-version: [ "3.8", "3.9", "3.10", "3.11"]
6768

68-
# name: Python ${{ matrix.python-version }} testing
69+
name: Python ${{ matrix.python-version }} testing
6970

70-
# steps:
71-
# - name: Check out repository code
72-
# uses: actions/checkout@v4
71+
steps:
72+
- name: Check out repository code
73+
uses: actions/checkout@v4
7374

74-
# # Setup Python (faster than using Python container)
75-
# - name: Setup Python
76-
# uses: actions/setup-python@v5
77-
# with:
78-
# python-version: ${{ matrix.python-version }}
79-
# - name: Lint code
80-
# uses: pre-commit/[email protected]
75+
# Setup Python (faster than using Python container)
76+
- name: Setup Python
77+
uses: actions/setup-python@v5
78+
with:
79+
python-version: ${{ matrix.python-version }}
80+
- name: Lint code
81+
uses: pre-commit/[email protected]
8182

82-
# - name: Install pipenv
83-
# run: |
84-
# python -m pip install --upgrade pipenv wheel
83+
- name: Install pipenv
84+
run: |
85+
python -m pip install --upgrade pipenv wheel
8586
86-
# - name: Install elasticsearch stac-fastapi
87-
# run: |
88-
# pip install ./stac_fastapi/elasticsearch[dev,server]
87+
- name: Install elasticsearch stac-fastapi
88+
run: |
89+
pip install ./stac_fastapi/elasticsearch[dev,server]
8990
90-
# - name: Install opensearch stac-fastapi
91-
# run: |
92-
# pip install ./stac_fastapi/opensearch[dev,server]
91+
- name: Install opensearch stac-fastapi
92+
run: |
93+
pip install ./stac_fastapi/opensearch[dev,server]
9394
94-
# - name: Install core library stac-fastapi
95-
# run: |
96-
# pip install ./stac_fastapi/core
95+
- name: Install core library stac-fastapi
96+
run: |
97+
pip install ./stac_fastapi/core
9798
98-
# - name: Run test suite against Elasticsearch 7.x
99-
# run: |
100-
# pipenv run pytest -svvv
101-
# env:
102-
# ENVIRONMENT: testing
103-
# ES_PORT: 9200
104-
# ES_HOST: 172.17.0.1
105-
# ES_USE_SSL: false
106-
# ES_VERIFY_CERTS: false
107-
# BACKEND: elasticsearch
99+
- name: Run test suite against Elasticsearch 7.x
100+
run: |
101+
pipenv run pytest -svvv
102+
env:
103+
ENVIRONMENT: testing
104+
ES_PORT: 9200
105+
ES_HOST: 172.17.0.1
106+
ES_USE_SSL: false
107+
ES_VERIFY_CERTS: false
108+
BACKEND: elasticsearch
108109

109-
# - name: Run test suite against Elasticsearch 8.x
110-
# run: |
111-
# pipenv run pytest -svvv
112-
# env:
113-
# ENVIRONMENT: testing
114-
# ES_PORT: 9400
115-
# ES_HOST: 172.17.0.1
116-
# ES_USE_SSL: false
117-
# ES_VERIFY_CERTS: false
118-
# BACKEND: elasticsearch
110+
- name: Run test suite against Elasticsearch 8.x
111+
run: |
112+
pipenv run pytest -svvv
113+
env:
114+
ENVIRONMENT: testing
115+
ES_PORT: 9400
116+
ES_HOST: 172.17.0.1
117+
ES_USE_SSL: false
118+
ES_VERIFY_CERTS: false
119+
BACKEND: elasticsearch
119120

120-
# - name: Run test suite against OpenSearch 2.11.1
121-
# run: |
122-
# pipenv run pytest -svvv
123-
# env:
124-
# ENVIRONMENT: testing
125-
# ES_PORT: 9202
126-
# ES_HOST: 172.17.0.1
127-
# ES_USE_SSL: false
128-
# ES_VERIFY_CERTS: false
129-
# BACKEND: opensearch
121+
- name: Run test suite against OpenSearch 2.11.1
122+
run: |
123+
pipenv run pytest -svvv
124+
env:
125+
ENVIRONMENT: testing
126+
ES_PORT: 9202
127+
ES_HOST: 172.17.0.1
128+
ES_USE_SSL: false
129+
ES_VERIFY_CERTS: false
130+
BACKEND: opensearch

‎CHANGELOG.md

+112-3
Original file line numberDiff line numberDiff line change
@@ -5,24 +5,133 @@ All notable changes to this project will be documented in this file.
55
The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
66
and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
77

8-
98
## [Unreleased]
109

1110
### Added
1211

12+
- Added explicit mapping for ID in `ES_COLLECTIONS_MAPPINGS` [#198](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/198)
13+
1314
### Changed
1415

15-
### Fixed
16+
- Removed database logic from core.py all_collections [#196](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/196)
17+
- Changed OpenSearch config ssl_version to SSLv23 [#200](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/200)
1618

19+
### Fixed
1720

1821
## [v2.0.0]
1922

2023
### Added
2124

25+
- Added core library package for common logic [#186](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/186)
26+
27+
### Changed
28+
29+
- Moved Elasticsearch and Opensearch backends into separate packages [#186](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/186)
30+
31+
### Fixed
32+
33+
- Allow additional top-level properties on collections [#191](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/191)
34+
35+
## [v1.1.0]
36+
37+
### Added
38+
39+
- Advanced comparison (LIKE, IN, BETWEEN) operators to the Filter extension [#178](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/178)
40+
- Collection update endpoint no longer delete all sub items [#177](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/177)
41+
- OpenSearch 2.11.1 support [#188](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/188)
42+
2243
### Changed
2344

45+
- Elasticsearch drivers from 7.17.9 to 8.11.0 [#169](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/169)
46+
- Collection update endpoint no longer delete all sub items [#177](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/177)
47+
48+
### Fixed
49+
50+
- Exclude unset fields in search response [#166](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/166)
51+
- Upgrade stac-fastapi to v2.4.9 [#172](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/172)
52+
- Set correct default filter-lang for GET /search requests [#179](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/179)
53+
54+
## [v1.0.0]
55+
56+
### Added
57+
58+
- Collection-level Assets to the CollectionSerializer [#148](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/148)
59+
- Pagination for /collections - GET all collections - route [#164](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/164)
60+
- Examples folder with example docker setup for running sfes from pip [#147](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/147)
61+
- GET /search filter extension queries [#163](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/163)
62+
- Added support for GET /search intersection queries [#158](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/158)
63+
64+
### Changed
65+
66+
- Update elasticsearch version from 8.1.3 to 8.10.4 in cicd, gh actions [#164](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/164)
67+
- Updated core stac-fastapi libraries to 2.4.8 from 2.4.3 [#151](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/151)
68+
- Use aliases on Elasticsearch indices, add number suffix in index name. [#152](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/152)
69+
2470
### Fixed
2571

72+
- Corrected the closing of client connections in ES index management functions [#132](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/132)
73+
- Corrected the automatic converstion of float values to int when building Filter Clauses [#135](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/135)
74+
- Do not index `proj:geometry` field as geo_shape [#154](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/154)
75+
- Remove unsupported characters from Elasticsearch index names [#153](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/153)
76+
- Fixed GET /search sortby requests [#25](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/25)
77+
78+
79+
## [v0.3.0]
80+
81+
### Added
82+
83+
- Added bbox and datetime parameters and functionality to item_collection [#127](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/127)
84+
- Added collection_id parameter to create_item function [#127](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/127)
85+
- Added item_id and collection_id to update_item [#127](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/127)
86+
- The default Collection objects index can be overridden by the `STAC_COLLECTIONS_INDEX` environment variable [#128](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/128)
87+
- The default Item objects index prefix can be overridden by the `STAC_ITEMS_INDEX_PREFIX` environment variable [#128](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/128)
88+
- Fields Extension [#129](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/129)
89+
- Support for Python 3.11 [#131](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/131)
90+
91+
### Changed
92+
93+
- Updated core stac-fastapi libraries to 2.4.3 from 2.3.0 [#127](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/127)
94+
95+
96+
## [v0.2.0]
97+
98+
### Added
99+
100+
- Filter Extension as GET with CQL2-Text and POST with CQL2-JSON,
101+
supporting the Basic CQL2 and Basic Spatial Operators conformance classes.
102+
- Added Elasticsearch local config to support snapshot/restore to local filesystem
103+
104+
### Fixed
105+
106+
- Fixed search intersects query.
107+
- Corrected the Sort and Query conformance class URIs.
108+
109+
### Changed
110+
111+
- Default to Python 3.10
112+
- Default to Elasticsearch 8.x
113+
- Collection objects are now stored in `collections` index rather than `stac_collections` index
114+
- Item objects are no longer stored in `stac_items`, but in indices per collection named `items_{collection_id}`
115+
- When using bulk ingest, items will continue to be ingested if any of them fail. Previously, the call would fail
116+
immediately if any items failed.
117+
118+
119+
## [v0.1.0]
120+
121+
### Changed
122+
123+
- Elasticsearch index mappings updated to be more thorough.
124+
- Endpoints that return items (e.g., /search) now sort the results by 'properties.datetime,id,collection'.
125+
Previously, there was no sort order defined.
126+
- Db_to_stac serializer moved to core.py for consistency as it existed in both core and database_logic previously.
127+
- Use genexp in execute_search and get_all_collections to return results.
128+
- Added db_to_stac serializer to item_collection method in core.py.
129+
26130

27131
[Unreleased]: <https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v2.0.0...main>
28-
[v2.0.0]: <https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v2.0.0>
132+
[v2.0.0]: <https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v1.1.0...v2.0.0>
133+
[v1.1.0]: <https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v1.0.0...v1.1.0>
134+
[v1.0.0]: <https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v0.3.0...v1.0.0>
135+
[v0.3.0]: <https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v0.2.0...v0.3.0>
136+
[v0.2.0]: <https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v0.1.0...v0.2.0>
137+
[v0.1.0]: <https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v0.1.0>

‎Dockerfile.deploy.es

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
FROM python:3.10-slim
2+
3+
RUN apt-get update && \
4+
apt-get -y upgrade && \
5+
apt-get -y install gcc && \
6+
apt-get clean && \
7+
rm -rf /var/lib/apt/lists/*
8+
9+
ENV CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
10+
11+
WORKDIR /app
12+
13+
COPY . /app
14+
15+
RUN pip install --no-cache-dir -e ./stac_fastapi/core
16+
RUN pip install --no-cache-dir ./stac_fastapi/elasticsearch[server]
17+
18+
EXPOSE 8080
19+
20+
CMD ["uvicorn", "stac_fastapi.elasticsearch.app:app", "--host", "0.0.0.0", "--port", "8080"]

0 commit comments

Comments
 (0)