split in 2 modules, base + modeler

Author: HviorForgeFlow

elasticsearch_base/__init__.py

@@ -0,0 +1,5 @@
+# Copyright 2017 Creu Blanca
+# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html).
+
+from . import components
+from . import models

elasticsearch_base/__manifest__.py

@@ -0,0 +1,26 @@
+# Copyright 2017 Creu Blanca
+# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html).
+
+{
+    'name': 'Elasticsearch Connector Base',
+    'version': '11.0.1.0.1',
+    'category': 'Elasticsearch connector',
+    'depends': [
+        'connector',
+    ],
+    'author': 'Creu Blanca',
+    'license': 'AGPL-3',
+    'summary': 'Elasticsearch connector base',
+    'data': [
+        'security/ir.model.access.csv',
+        'views/menu.xml',
+        'views/elasticsearch_host_views.xml',
+        'views/elasticsearch_index_views.xml',
+    ],
+    'external_dependencies': {
+        'python': [
+            'elasticsearch'
+        ],
+    },
+    'installable': True,
+}

elasticsearch_base/components/__init__.py

@@ -0,0 +1,7 @@
+# Copyright 2017 Creu Blanca
+# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html).
+
+from . import core
+from . import binder
+from . import exporter
+from . import listener

elasticsearch_base/components/binder.py

@@ -0,0 +1,10 @@
+# Copyright 2017 Creu Blanca
+# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html).
+
+from odoo.addons.component.core import Component
+
+
+class ElasticsearchModelBinder(Component):
+    _name = 'elasticsearch.binder'
+    _inherit = ['base.binder', 'base.elasticsearch.connector']
+    _apply_on = False

elasticsearch_base/components/core.py

@@ -0,0 +1,14 @@
+# Copyright 2017 Creu Blanca
+# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html).
+
+from odoo.addons.component.core import AbstractComponent
+
+
+class BaseElasticsearchConnectorComponent(AbstractComponent):
+    """ Base elasticsearch Connector Component
+    All components of this connector should inherit from it.
+    """
+
+    _name = 'base.elasticsearch.connector'
+    _inherit = 'base.connector'
+    _collection = 'elasticsearch.index'

elasticsearch_base/components/exporter.py

@@ -0,0 +1,118 @@
+# Copyright 2017 Creu Blanca
+# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html).
+
+import logging
+import psycopg2
+import json
+from datetime import datetime
+
+import odoo
+from odoo.addons.component.core import Component
+from odoo.addons.connector.exception import RetryableJobError
+
+_logger = logging.getLogger(__name__)
+try:
+    import elasticsearch
+except (ImportError, IOError) as err:
+    _logger.debug(err)
+
+ISO_FORMAT = '%Y-%m-%dT%H:%M:%S.%f'
+
+
+class ElasticsearchBaseExporter(Component):
+    """ Base exporter for the Elasticsearch """
+
+    _name = 'elasticsearch.document.exporter'
+    _inherit = ['base.exporter', 'base.elasticsearch.connector']
+    _usage = 'record.exporter'
+    _apply_on = ['elasticsearch.document']
+    _exporter_failure_timeout = 2
+
+    def _lock(self, document):
+        """ Lock the binding record.
+        Lock the binding record so we are sure that only one export
+        job is running for this record if concurrent jobs have to export the
+        same record.
+        When concurrent jobs try to export the same record, the first one
+        will lock and proceed, the others will fail to lock and will be
+        retried later.
+        This behavior works also when the export becomes multilevel
+        with :meth:`_export_dependencies`. Each level will set its own lock
+        on the binding record it has to export.
+        """
+        sql = ("SELECT id FROM %s WHERE ID = %%s FOR UPDATE NOWAIT" %
+               self.model._table)
+        try:
+            self.env.cr.execute(sql, (document.id,),
+                                log_exceptions=False)
+        except psycopg2.OperationalError:
+            _logger.info('A concurrent job is already exporting the same '
+                         'record (%s with id %s). Job delayed later.',
+                         self.model._name, document.id)
+            raise RetryableJobError(
+                'A concurrent job is already exporting the same record '
+                '(%s with id %s). The job will be retried later.' %
+                (self.model._name, document.id),
+                seconds=self._exporter_failure_timeout)
+
+    def es_create(self, document, *args, **kwargs):
+        self._lock(document)
+        index = document.index_id.index
+        es = elasticsearch.Elasticsearch(
+            hosts=document.index_id.get_hosts())
+        data = json.dumps(kwargs['data'])
+        es.index(index, '_doc', id=document.id, body=data)
+        document.with_context(no_elasticserach_sync=True).write({
+            'sync_date': kwargs['sync_date']
+        })
+        if not odoo.tools.config['test_enable']:
+            self.env.cr.commit()  # pylint: disable=E8102
+        self._after_export()
+        return True
+
+    def es_unlink(self, id, index, *args, **kwargs):
+        """ Run the synchronization
+        :param binding: binding record to export
+        """
+        es = elasticsearch.Elasticsearch(hosts=index.get_hosts())
+        es.delete(index.index, '_doc', id)
+        self._after_export()
+        return True
+
+    def es_write(self, document, *args, **kwargs):
+        """ Run the synchronization
+        :param binding: binding record to export
+        """
+        self._lock(document)
+        sync_date = datetime.strptime(kwargs['sync_date'], ISO_FORMAT)
+        if (
+            not document.sync_date or
+            sync_date >= datetime.strptime(document.sync_date, ISO_FORMAT)
+        ):
+            index = document.index_id.index
+            es = elasticsearch.Elasticsearch(
+                hosts=document.index_id.get_hosts())
+            data = json.dumps(kwargs['data'])
+            es.index(index, '_doc', id=document.id, body=data)
+            document.with_context(no_elasticserach_sync=True).write({
+                'sync_date': kwargs['sync_date']
+            })
+        else:
+            _logger.info(
+                'Record from %s with id %s has already been sended (%s), so it'
+                ' is deprecated ' % (
+                    self.model._name, document.id, kwargs['sync_date']
+                )
+            )
+        # Commit so we keep the external ID when there are several
+        # exports (due to dependencies) and one of them fails.
+        # The commit will also release the lock acquired on the binding
+        # record
+        if not odoo.tools.config['test_enable']:
+            self.env.cr.commit()  # pylint: disable=E8102
+        self._after_export()
+        return True
+
+    def _after_export(self):
+        """ Can do several actions after exporting a record"""
+        pass

elasticsearch_base/components/listener.py

@@ -0,0 +1,32 @@
+# Copyright 2017 Creu Blanca
+# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html).
+
+from odoo.addons.component.core import Component
+from datetime import datetime
+
+
+class ElasticsearchDocumentListener(Component):
+    _name = 'elasticsearch.document.listener'
+    _inherit = 'base.event.listener'
+    _apply_on = 'elasticsearch.document'
+
+    def on_record_create(self, record, fields):
+        if self.env.context.get('no_elasticserach_sync', False):
+            return
+        for rec in record:
+            rec.with_delay().export_create(datetime.now().isoformat())
+
+    def on_record_write(self, record, fields):
+        if self.env.context.get('no_elasticserach_sync', False):
+            return
+        for rec in record:
+            rec.with_delay().export_update(datetime.now().isoformat())
+
+    def on_record_unlink(self, record):
+        if self.env.context.get('no_elasticserach_sync', False):
+            return
+        if self.env.context.get('no_elasticsearch_delay', False):
+            for rec in record:
+                rec.export_delete(rec.id, rec.index_id)
+        for rec in record:
+            rec.with_delay()

elasticsearch_base/models/__init__.py

@@ -0,0 +1,7 @@
+# Copyright 2017 Creu Blanca
+# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html).
+
+from . import elasticsearch_document
+from . import elasticsearch_host
+from . import elasticsearch_index
+from . import elasticsearch_model

elasticsearch_base/models/elasticsearch_document.py

@@ -0,0 +1,53 @@
+# Copyright 2017 Creu Blanca
+# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html).
+
+from datetime import datetime
+
+from odoo import api, models, fields
+from odoo.addons.queue_job.job import job
+
+
+class ElasticsearchDocument(models.Model):
+    _name = 'elasticsearch.document'
+    _inherit = 'external.binding'
+    _description = 'Elasticsearch abstract Document'
+
+    index_id = fields.Many2one(
+        comodel_name='elasticsearch.index',
+        string='Index',
+        required=True,
+        ondelete='restrict',
+    )
+    sync_date = fields.Char()
+
+    @job(default_channel='root.elasticsearch')
+    @api.multi
+    def export_create(self, date=datetime.now().isoformat()):
+        self.ensure_one()
+        with self.index_id.work_on(self._name) as work:
+            exporter = work.component(usage='record.exporter')
+            return exporter.es_create(
+                self,
+                data=self.get_document_values(),
+                sync_date=date)
+
+    @job(default_channel='root.elasticsearch')
+    @api.multi
+    def export_update(self, date=datetime.now().isoformat()):
+        self.ensure_one()
+        with self.index_id.work_on(self._name) as work:
+            exporter = work.component(usage='record.exporter')
+            return exporter.es_write(
+                self,
+                data=self.get_document_values(),
+                sync_date=date)
+
+    @job(default_channel='root.elasticsearch')
+    @api.multi
+    def export_delete(self, id, index):
+        with index.work_on(self._name) as work:
+            exporter = work.component(usage='record.exporter')
+            return exporter.es_unlink(id, index)
+
+    def get_document_values(self):
+        return {}

elasticsearch_base/models/elasticsearch_host.py

@@ -0,0 +1,17 @@
+# Copyright 2018 Creu Blanca
+# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html).
+
+from odoo import models, fields
+
+
+class ElasticsearchHost(models.Model):
+    _name = 'elasticsearch.host'
+    _description = 'ElasticSearch Host'
+
+    host = fields.Char(
+        required=True
+    )
+    port = fields.Integer(
+        required=True,
+        default=9200,
+    )

elasticsearch_base/models/elasticsearch_index.py

@@ -0,0 +1,97 @@
+# Copyright 2017 Creu Blanca
+# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html).
+import logging
+from odoo import api, fields, models
+from odoo.tools import safe_eval
+import json
+
+_logger = logging.getLogger(__name__)
+try:
+    import elasticsearch
+except (ImportError, IOError) as err:
+    _logger.debug(err)
+
+
+class ElasticsearchIndex(models.Model):
+    _name = 'elasticsearch.index'
+    _description = 'Elasticsearch Index'
+    _inherit = 'connector.backend'
+
+    name = fields.Char(
+        required=True,
+        readonly=True,
+        states={'draft': [('readonly', False)]},
+    )
+    state = fields.Selection([
+        ('draft', 'Draft'),
+        ('posted', 'Posted'),
+        ('cancelled', 'Cancelled')
+    ], required=True, default='draft', readonly=True)
+    index = fields.Char(
+        required=True,
+        readonly=True,
+        states={'draft': [('readonly', False)]},
+    )
+    host_ids = fields.Many2many(
+        'elasticsearch.host',
+        required=True,
+        readonly=True,
+        states={'draft': [('readonly', False)]},
+    )
+    document_ids = fields.One2many(
+        'elasticsearch.document',
+        inverse_name='index_id',
+        readonly=True
+    )
+    type = fields.Selection(
+        [],
+        required=True,
+        readonly=True,
+        states={'draft': [('readonly', False)]},
+    )
+
+    def _post_values(self):
+        return {'state': 'posted'}
+
+    def _get_index_template(self):
+        return {
+            "settings": {"index.mapping.ignore_malformed": True}
+        }
+
+    def _post(self):
+        es = elasticsearch.Elasticsearch(hosts=self.get_hosts())
+        es.indices.create(
+            self.index,
+            body=json.dumps(self._get_index_template()))
+
+    @api.multi
+    def post(self):
+        self.ensure_one()
+        self._post()
+        self.write(self._post_values())
+
+    def _reset_index(self):
+        self.ensure_one()
+        es = elasticsearch.Elasticsearch(hosts=self.get_hosts())
+        self.document_ids.with_context(no_elasticserach_sync=True).unlink()
+        es.indices.delete(index=self.index, ignore=[400, 404])
+        self.state = 'cancelled'
+
+    def _draft_values(self):
+        return {'state': 'draft'}
+
+    @api.multi
+    def restore(self):
+        self.write(self._draft_values())
+
+    def _cancel_values(self):
+        return {'state': 'cancelled'}
+
+    @api.multi
+    def cancel(self):
+        self.ensure_one()
+        self._reset_index()
+        self.write(self._cancel_values())
+
+    def get_hosts(self):
+        return [{"host": r.host, "port": r.port} for r in self.host_ids]