initial commit

Author: liorgenesort

.gitignore

@@ -0,0 +1,6 @@
+venv/
+.idea/
+__pycache__
+flaskr/__pycache__
+migrations/__pycache__
+

Dockerfile

@@ -0,0 +1,22 @@
+FROM ubuntu:18.04
+
+RUN apt-get update -y && \
+    apt-get install -y python3-pip python3-dev python3
+
+# We copy just the requirements.txt first to leverage Docker cache
+COPY ./requirements.txt /app/requirements.txt
+
+WORKDIR /app
+
+RUN pip3 install -r requirements.txt
+
+COPY . /app
+
+ENV FLASK_APP=run.py
+ENV FLASK_DEBUG=True
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
+
+#ENTRYPOINT "/usr/bin/python3"
+
+CMD [ "flask", "run", "--host=0.0.0.0" ]

README.md

@@ -0,0 +1,49 @@
+### Installation
+clone the repository
+
+    cd python-flask-elastic
+    docker-compose up -d
+
+Run the migrations for elasticsearch
+
+    cd migrations
+    python3 migration_runner.py    
+
+### Usage
+To add a pokemon to the database:
+```
+curl --header "Content-Type: application/json" -d "{
+   \"pokadex_id\": 25,
+   \"name\": \"Stam\",   
+   \"nickname\": \"Lior Ha Gever\", 
+   \"level\": 60,
+   \"type\": \"ELECTRIC\",
+   \"skills\": [
+       \"Tail Whip\"
+   ]
+}" localhost:5000/new_pokemon
+```
+To search (autocomplete) for a pokemon, browse to 
+http://localhost:5000/autocomplete/<search_term>
+### Requirements
+* docker-compose
+* python3.6+
+
+``` 
+
+
+curl --header "Content-Type: application/json" -d "{                                         
+   \"pokadex_id\": 26,
+   \"name\": \"Pikachu\",
+   \"nickname\": \"Baruh Ha Gever\",
+   \"level\": 60,
+   \"type\": \"ELECTRIC\",
+   \"skills\": [
+       \"Tail Whip\"
+   ]
+}"
+```
+
+### Notes
+The migration script should wait until the elasticsearch 
+warms up (estimated: 25 seconds)

configs/app_config.yaml

@@ -0,0 +1,4 @@
+PORT: 5000
+ES_CONNECTION:
+  HOST: elasticsearch1
+  PORT: 9200

configs/config.py

@@ -0,0 +1,7 @@
+import yaml
+
+
+def load_config(path: str):
+    with open(path, 'r') as f:
+        cfg = yaml.safe_load(f)
+        return cfg

docker-compose.yaml

@@ -0,0 +1,94 @@
+
+version: '3.7'
+services:
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.4.0
+    container_name: elasticsearch1
+    environment:
+      - node.name=elasticsearch1
+      - cluster.name=docker-cluster
+      - cluster.initial_master_nodes=elasticsearch1
+      - bootstrap.memory_lock=true
+      - "ES_JAVA_OPTS=-Xms1024M -Xmx1024M"
+      - http.cors.enabled=true
+      - http.cors.allow-origin=*
+      - network.host=_eth0_
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9200"]
+      interval: 1m30s
+      timeout: 10s
+      retries: 3
+      start_period: 40s
+    ulimits:
+      nproc: 65535
+      memlock:
+        soft: -1
+        hard: -1
+    cap_add:
+      - ALL
+    # privileged: true
+    deploy:
+      replicas: 1
+      update_config:
+        parallelism: 1
+        delay: 10s
+      resources:
+        limits:
+          cpus: '1'
+          memory: 256M
+        reservations:
+          cpus: '1'
+          memory: 256M
+      restart_policy:
+        condition: on-failure
+        delay: 5s
+        max_attempts: 3
+        window: 10s
+    volumes:
+      - type: volume
+        source: logs
+        target: /var/log
+      - type: volume
+        source: esdata3
+        target: /usr/share/elasticsearch/data
+    networks:
+      esnet:
+        aliases:
+          - elasticsearch
+    ports:
+      - 9200:9200
+      - 9300:9300
+  migrate:
+    image: webapp-python
+    depends_on:
+     - elasticsearch
+    networks:
+      - esnet
+    working_dir: /app/migrations
+    command: [ "python3", "migration_runner.py" ]
+  flask:
+    image: webapp-python
+    depends_on:
+      - elasticsearch
+    networks:
+      - esnet
+    build:
+      context: .
+      dockerfile: Dockerfile
+    volumes:
+      - .:/app
+    environment:
+      - FLASK_APP=/app/run.py
+      - FLASK_DEBUG=True
+      - LC_ALL=C.UTF-8
+      - LANG=C.UTF-8
+    ports:
+      - 5000:5000
+    command: [ "flask", "run", "--host=0.0.0.0" ]
+volumes:
+  esdata3:
+  logs:
+
+networks:
+  esnet:
+    driver: bridge

flaskr/routes.py

@@ -0,0 +1,32 @@
+from run import app, es
+from flask import request, jsonify
+import flaskr.utils as utils
+
+POKEMON_INDEX = 'pokemon'
+
+@app.route('/', methods=['GET'])
+def index():
+    return 'This is the index page\n'
+
+
+@app.route('/new_pokemon', methods=['POST'])
+def add_pokemon():
+    if not utils.valid_new_pokemon_schema(request.json):
+        return 'Bad Request\n'
+    pokemon_id, pokemon_body = utils.valid_pokemon_dict_to_id_body(request.json)
+    result = es.index(index=POKEMON_INDEX, id=pokemon_id, body=pokemon_body)
+    return f'New Pokemon Added\n{jsonify(result)}\n'
+
+
+@app.route('/autocomplete/<string:pokemon>')
+def auto_complete(pokemon):
+    fields = ['nickname', 'name', 'skills']
+    results = es.search(index=POKEMON_INDEX,
+                        body={'query': {'multi_match': {'fields': fields, 'query': pokemon, }}})
+    return jsonify(results['hits']['hits'])
+
+
+@app.route('/query_pokemon', methods=['POST'])
+def query():
+    results = es.get(index=POKEMON_INDEX, id=int(request.json.get('id', 0)))
+    return jsonify(results['_source'])

flaskr/utils.py

@@ -0,0 +1,75 @@
+from cerberus import validator
+
+_VALID_POKEMON_LEVELS = [10 * x for x in range(10)]
+
+
+def _valid_level(field, value, error):
+    if value not in _VALID_POKEMON_LEVELS:
+        error(field, "Invalid Pokemon Level")
+
+
+_VALID_POKEMON_TYPES = 'ELECTRIC GROUND FIRE WATER WIND PSYCHIC GRASS'.split()
+
+
+def _valid_type(field, value, error):
+    if value not in _VALID_POKEMON_TYPES:
+        error(field, "Invalid Pokemon Type")
+
+
+_NEW_POKEMON_SCHEMA = {'pokadex_id': {'required': True, 'type': 'integer'},
+                       'name': {'required': True, 'type': 'string'},
+                       'nickname': {'required': True, 'type': 'string'},
+                       'level': {'required': True, 'check_with': _valid_level},
+                       'type': {'required': True, 'check_with': _valid_type},
+                       'skills': {'required': True, 'type': 'list', 'schema': {'type': 'string'}}
+                       }
+
+_POKEMON_INDEX_SCHEMA = {
+    'settings': {
+     "analysis": {
+          "filter": {
+            "autocomplete_filter": {
+              "type": "edge_ngram",
+              "min_gram": 1,
+              "max_gram": 20
+            }
+          },
+          "analyzer": {
+            "autocomplete": {
+              "type": "custom",
+              "tokenizer": "standard",
+              "filter": [
+                "lowercase",
+                "autocomplete_filter"
+              ]
+            }
+          }
+        }
+    },
+    'mappings': {
+        'properties': {
+            'pokadex_id': {'type': 'integer'},
+            'name': {'type': 'completion', 'analyzer': 'autocomplete'},
+            'nickname': {'type': 'completion', 'analyzer': 'autocomplete'},
+            'level': {'type': 'integer'},
+            'type': {'type': 'text'},
+            'skills': {'type': 'text'}
+        }
+    }
+}
+
+
+def valid_new_pokemon_schema(dictionary):
+    val = validator.Validator(_NEW_POKEMON_SCHEMA)
+    return val.validate(dictionary)
+
+
+def valid_pokemon_dict_to_id_body(dictionary):
+    pokemon_id = dictionary.get('pokadex_id')
+    return int(pokemon_id), dictionary
+
+
+def generate_index(elastic_obj):
+    if elastic_obj.indices.exists(index='pokemon'):
+        elastic_obj.indices.delete(index='pokemon')
+    elastic_obj.indices.create(index='pokemon', body=_POKEMON_INDEX_SCHEMA)

lib/models/pokemon.py

@@ -0,0 +1,36 @@
+from cerberus import validator
+
+_VALID_POKEMON_LEVELS = [10 * x for x in range(10)]
+
+
+def _valid_level(field, value, error):
+    if value not in _VALID_POKEMON_LEVELS:
+        error(field, "Invalid Pokemon Level")
+
+
+_VALID_POKEMON_TYPES = 'ELECTRIC GROUND FIRE WATER WIND PSYCHIC GRASS'.split()
+
+
+def _valid_type(field, value, error):
+    if value not in _VALID_POKEMON_TYPES:
+        error(field, "Invalid Pokemon Type")
+
+
+_NEW_POKEMON_SCHEMA = {'pokadex_id': {'required': True, 'type': 'integer'},
+                       'name': {'required': True, 'type': 'string'},
+                       'nickname': {'required': True, 'type': 'string'},
+                       'level': {'required': True, 'check_with': _valid_level},
+                       'type': {'required': True, 'check_with': _valid_type},
+                       'skills': {'required': True, 'type': 'list', 'schema': {'type': 'string'}}
+                       }
+
+
+def valid_new_pokemon_schema(dictionary):
+    val = validator.Validator(_NEW_POKEMON_SCHEMA)
+    return val.validate(dictionary)
+
+
+def valid_pokemon_dict_to_id_body(dictionary):
+    pokemon_id = dictionary.get('pokadex_id')
+    return int(pokemon_id), dictionary
+

migrations/1574608301693-set-pokemon-schema.es-migration.py

@@ -0,0 +1,45 @@
+from es_migration_base import BaseESMigration
+
+
+class Migration(BaseESMigration):
+
+    def __init__(self, es_object):
+        super().__init__(es_object=es_object, es_index='pokemon')
+        self.schema = {
+            'settings': {
+                "analysis": {
+                    "filter": {
+                        "autocomplete_filter": {
+                            "type": "edge_ngram",
+                            "min_gram": 1,
+                            "max_gram": 20
+                        }
+                    },
+                    "analyzer": {
+                        "autocomplete": {
+                            "type": "custom",
+                            "tokenizer": "standard",
+                            "filter": [
+                                "lowercase",
+                                "autocomplete_filter"
+                            ]
+                        }
+                    }
+                }
+            },
+            'mappings': {
+                'properties': {
+                    'pokadex_id': {'type': 'integer'},
+                    'name': {'type': 'completion', 'analyzer': 'autocomplete'},
+                    'nickname': {'type': 'completion', 'analyzer': 'autocomplete'},
+                    'level': {'type': 'integer'},
+                    'type': {'type': 'text'},
+                    'skills': {'type': 'completion', 'analyzer': 'autocomplete'}
+                }
+            }
+        }
+
+    def execute(self):
+        if self._es_object.indices.exists(index='pokemon'):
+            self._es_object.indices.delete(index='pokemon')
+        self._es_object.indices.create(index='pokemon', body=self.schema)

migrations/__init__.py

@@ -0,0 +1,11 @@
+from abc import ABC, abstractmethod
+
+
+class BaseESMigration(ABC):
+    def __init__(self, es_object, es_index):
+        self._es_object = es_object
+        self.es_index = es_index
+
+    @abstractmethod
+    def execute(self):
+        pass