First version

etobella · etobella · commit 4cdcb3e9d5f5 · 2018-06-06T14:09:35.000+02:00
diff --git a/dependencies.txt b/dependencies.txt
@@ -0,0 +1 @@
+elasticsearch
diff --git a/elasticsearch_connector_base/__init__.py b/elasticsearch_connector_base/__init__.py
@@ -0,0 +1,5 @@
+# Copyright 2017 Creu Blanca
+# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html).
+
+from . import components
+from . import models
diff --git a/elasticsearch_connector_base/__manifest__.py b/elasticsearch_connector_base/__manifest__.py
@@ -0,0 +1,24 @@
+# Copyright 2017 Creu Blanca
+# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html).
+
+{
+    'name': 'Elasticsearch Connector Base',
+    'version': '11.0.1.0.1',
+    'category': 'Elasticsearch connector',
+    'depends': [
+        'connector',
+    ],
+    'author': 'Creu Blanca',
+    'license': 'AGPL-3',
+    'summary': 'Elasticsearch connector base',
+    'data': [
+        'data/backend.xml',
+        'security/ir.model.access.csv',
+    ],
+    'external_dependencies': {
+        'python': [
+            'elasticsearch'
+        ],
+    },
+    'installable': True,
+}
diff --git a/elasticsearch_connector_base/components/__init__.py b/elasticsearch_connector_base/components/__init__.py
@@ -0,0 +1,7 @@
+# Copyright 2017 Creu Blanca
+# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html).
+
+from . import core
+from . import binder
+from . import exporter
+from . import listener
diff --git a/elasticsearch_connector_base/components/binder.py b/elasticsearch_connector_base/components/binder.py
@@ -0,0 +1,10 @@
+# Copyright 2017 Creu Blanca
+# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html).
+
+from odoo.addons.component.core import Component
+
+
+class ElasticsearchModelBinder(Component):
+    _name = 'elasticsearch.binder'
+    _inherit = ['base.binder', 'base.elasticsearch.connector']
+    _apply_on = False
diff --git a/elasticsearch_connector_base/components/core.py b/elasticsearch_connector_base/components/core.py
@@ -0,0 +1,14 @@
+# Copyright 2017 Creu Blanca
+# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html).
+
+from odoo.addons.component.core import AbstractComponent
+
+
+class BaseElasticsearchConnectorComponent(AbstractComponent):
+    """ Base elasticsearch Connector Component
+    All components of this connector should inherit from it.
+    """
+
+    _name = 'base.elasticsearch.connector'
+    _inherit = 'base.connector'
+    _collection = 'elasticsearch.backend'
diff --git a/elasticsearch_connector_base/components/exporter.py b/elasticsearch_connector_base/components/exporter.py
@@ -0,0 +1,141 @@
+# Copyright 2017 Creu Blanca
+# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html).
+
+import logging
+import psycopg2
+import simplejson
+import requests
+from datetime import datetime
+
+import odoo
+from odoo.addons.component.core import AbstractComponent
+from odoo.addons.connector.exception import RetryableJobError
+import elasticsearch
+
+ISO_FORMAT = '%Y-%m-%dT%H:%M:%S.%f'
+_logger = logging.getLogger(__name__)
+
+
+class ElasticsearchBaseExporter(AbstractComponent):
+    """ Base exporter for the Elasticsearch """
+
+    _name = 'elasticsearch.base.exporter'
+    _inherit = ['base.exporter', 'base.elasticsearch.connector']
+    _usage = 'record.exporter'
+    _exporter_failure_timeout = 2
+
+    def _lock(self, binding):
+        """ Lock the binding record.
+        Lock the binding record so we are sure that only one export
+        job is running for this record if concurrent jobs have to export the
+        same record.
+        When concurrent jobs try to export the same record, the first one
+        will lock and proceed, the others will fail to lock and will be
+        retried later.
+        This behavior works also when the export becomes multilevel
+        with :meth:`_export_dependencies`. Each level will set its own lock
+        on the binding record it has to export.
+        """
+        sql = ("SELECT id FROM %s WHERE ID = %%s FOR UPDATE NOWAIT" %
+               self.model._table)
+        try:
+            self.env.cr.execute(sql, (binding.id,),
+                                log_exceptions=False)
+        except psycopg2.OperationalError:
+            _logger.info('A concurrent job is already exporting the same '
+                         'record (%s with id %s). Job delayed later.',
+                         self.model._name, binding.id)
+            raise RetryableJobError(
+                'A concurrent job is already exporting the same record '
+                '(%s with id %s). The job will be retried later.' %
+                (self.model._name, binding.id),
+                seconds=self._exporter_failure_timeout)
+
+    def create(self, binding, *args, **kwargs):
+        self._lock(binding)
+        index = binding.index
+        doc_type = binding.doc_type
+        es = elasticsearch.Elasticsearch(
+            hosts=binding.backend_id.get_hosts())
+        data = simplejson.dumps(kwargs['data'])
+        es.create(index, doc_type, binding.id, body=data)
+        binding.sync_date = kwargs['sync_date']
+        if not odoo.tools.config['test_enable']:
+            self.env.cr.commit()  # noqa
+        self._after_export()
+        return True
+
+    def delete(self, binding, *args, **kwargs):
+        """ Run the synchronization
+        :param binding: binding record to export
+        """
+        self._lock(binding)
+        sync_date = datetime.strptime(kwargs['sync_date'], ISO_FORMAT)
+        if (
+            not binding.sync_date or
+            sync_date >= datetime.strptime(binding.sync_date, ISO_FORMAT)
+        ):
+            index = binding.index
+            doc_type = binding.doc_type
+            es = elasticsearch.Elasticsearch(
+                hosts=binding.backend_id.get_hosts())
+            es.delete(index, doc_type, binding.id)
+            binding.sync_date = kwargs['sync_date']
+            if not odoo.tools.config['test_enable']:
+                self.env.cr.commit()  # noqa
+            binding.sync_date = kwargs['sync_date']
+        else:
+            _logger.info(
+                'Record from %s with id %s has already been sended (%s), so it'
+                ' is deprecated ' % (
+                    self.model._name, binding.id, kwargs['sync_date']
+                )
+            )
+        # Commit so we keep the external ID when there are several
+        # exports (due to dependencies) and one of them fails.
+        # The commit will also release the lock acquired on the binding
+        # record
+        if not odoo.tools.config['test_enable']:
+            self.env.cr.commit()  # noqa
+        self._after_export()
+        return True
+
+    def update(self, binding, *args, **kwargs):
+        """ Run the synchronization
+        :param binding: binding record to export
+        """
+        self._lock(binding)
+        sync_date = datetime.strptime(kwargs['sync_date'], ISO_FORMAT)
+        if (
+            not binding.sync_date or
+            sync_date >= datetime.strptime(binding.sync_date, ISO_FORMAT)
+        ):
+            index = binding.index
+            doc_type = binding.doc_type
+            es = elasticsearch.Elasticsearch(
+                hosts=binding.backend_id.get_hosts())
+            data = simplejson.dumps(kwargs['data'])
+            es.update(index, doc_type, binding.id, body=data)
+            binding.sync_date = kwargs['sync_date']
+            if not odoo.tools.config['test_enable']:
+                self.env.cr.commit()  # noqa
+            binding.sync_date = kwargs['sync_date']
+        else:
+            _logger.info(
+                'Record from %s with id %s has already been sended (%s), so it'
+                ' is deprecated ' % (
+                    self.model._name, binding.id, kwargs['sync_date']
+                )
+            )
+        # Commit so we keep the external ID when there are several
+        # exports (due to dependencies) and one of them fails.
+        # The commit will also release the lock acquired on the binding
+        # record
+        if not odoo.tools.config['test_enable']:
+            self.env.cr.commit()  # noqa
+        self._after_export()
+        return True
+
+    def _after_export(self):
+        """ Can do several actions after exporting a record"""
+        pass
diff --git a/elasticsearch_connector_base/components/listener.py b/elasticsearch_connector_base/components/listener.py
@@ -0,0 +1,32 @@
+# Copyright 2017 Creu Blanca
+# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html).
+
+from odoo.addons.component.core import AbstractComponent
+from datetime import datetime
+
+
+class AbstractElasticsearchListener(AbstractComponent):
+    _name = 'abstract.elasticsearch.listener'
+    _inherit = 'base.event.listener'
+    _apply_on = False
+
+    def on_record_create(self, record):
+        for rec in record:
+            for backend in rec.get_backends():
+                for vals in rec.get_binds(backend):
+                    binding = self.env[rec.get_binding_model()].create(vals)
+                    binding.with_delay().export_create(
+                        datetime.now().isoformat())
+
+    def on_record_write(self, record):
+        for rec in record:
+            if rec._fields.get('elasticsearch_bind_ids'):
+                for binding in rec.elasticsearch_bind_ids:
+                    binding.with_delay().export_update(
+                        datetime.now().isoformat())
+
+    def on_record_unlink(self, record):
+        for rec in record:
+            for binding in rec.elasticsearch_bind_ids:
+                binding.with_delay().binding.export_delete(
+                    datetime.now().isoformat())
diff --git a/elasticsearch_connector_base/data/backend.xml b/elasticsearch_connector_base/data/backend.xml
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<odoo>
+    <record id="elasticsearch" model="elasticsearch.backend">
+        <field name="name">Elasticsearch</field>
+    </record>
+</odoo>
diff --git a/elasticsearch_connector_base/models/__init__.py b/elasticsearch_connector_base/models/__init__.py
@@ -0,0 +1,6 @@
+# Copyright 2017 Creu Blanca
+# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html).
+
+from . import backend
+from . import binding
+from . import elasticsearch_model
diff --git a/elasticsearch_connector_base/models/backend.py b/elasticsearch_connector_base/models/backend.py
@@ -0,0 +1,24 @@
+# Copyright 2017 Creu Blanca
+# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html).
+
+from odoo import api, fields, models
+from odoo.addons.server_environment import serv_config
+
+BUS_SECTION = 'elasticsearch'
+
+
+class ElasticsearchBackend(models.Model):
+    _name = 'elasticsearch.backend'
+    _description = 'Elasticsearch Backend'
+    _inherit = 'connector.backend'
+
+    name = fields.Char(
+        readonly=True
+    )
+    destination = fields.Char(
+        string='Destination',
+    )
+    port = fields.Integer()
+
+    def get_hosts(self):
+        return [{"host": self.destination, "port": self.port}]
diff --git a/elasticsearch_connector_base/models/binding.py b/elasticsearch_connector_base/models/binding.py
@@ -0,0 +1,71 @@
+# Copyright 2017 Creu Blanca
+# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html).
+
+from datetime import datetime
+
+from odoo import api, models, fields
+from odoo.addons.queue_job.job import job
+
+
+class ElasticsearchBinding(models.AbstractModel):
+    _name = 'elasticsearch.binding'
+    _inherit = 'external.binding'
+    _description = 'Elasticsearch abstract Binding'
+    _elasticsearch_fields = []
+
+    odoo_id = fields.Many2one(
+        required=True,
+        comodel_name='elasticsearch.model',
+    )
+    index = fields.Char()
+    doc_type = fields.Char()
+    backend_id = fields.Many2one(
+        comodel_name='elasticsearch.backend',
+        string='CB Backend',
+        required=True,
+        ondelete='restrict',
+    )
+    elasticsearch_document_id = fields.Integer()
+    sync_date = fields.Char()
+
+    @job(default_channel='root.elasticsearch')
+    @api.multi
+    def export_create(self, date=datetime.now().isoformat()):
+        self.ensure_one()
+        with self.backend_id.work_on(self._name) as work:
+            exporter = work.component(usage='record.exporter')
+            return exporter.create(
+                self,
+                data=self.get_binding_values(),
+                sync_date=date)
+
+    @job(default_channel='root.elasticsearch')
+    @api.multi
+    def export_update(self, date=datetime.now().isoformat()):
+        self.ensure_one()
+        with self.backend_id.work_on(self._name) as work:
+            exporter = work.component(usage='record.exporter')
+            return exporter.update(
+                self,
+                data=self.get_binding_values(),
+                sync_date=date)
+
+    @job(default_channel='root.elasticsearch')
+    @api.multi
+    def export_delete(self, date=datetime.now().isoformat()):
+        self.ensure_one()
+        with self.backend_id.work_on(self._name) as work:
+            exporter = work.component(usage='record.exporter')
+            return exporter.delete(
+                self,
+                document_id=self.elasticsearch_document_id,
+                sync_date=date
+            )
+
+    @api.model
+    def get_fields(self):
+        return ['id', 'write_date', 'write_uid', 'create_date', 'create_uid']
+
+    def get_binding_values(self):
+        flds = list(set(self._elasticsearch_fields + self.get_fields()))
+        return self.odoo_id.read(flds, load=False)[0]
diff --git a/elasticsearch_connector_base/models/bus_model.py b/elasticsearch_connector_base/models/bus_model.py
@@ -0,0 +1,39 @@
+# Copyright 2017 Creu Blanca
+# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html).
+
+from odoo import api, fields, models
+
+
+class ElasticsearchModel(models.AbstractModel):
+    _name = 'elasticsearch.model'
+    _binding_name = False
+
+    elasticsearch_bind_ids = fields.One2many(
+        comodel_name='elasticsearch.binding',
+        inverse_name='odoo_id',
+        string='Elasticsearch Bindings'
+    )
+
+    @api.model
+    def get_backends(self):
+        return self.env['elasticsearch.backend'].search([])
+
+    @api.model
+    def get_binding_model(self):
+        return 'elasticsearch.binding.' + self._name
+
+    def bind_values(self, backend, elasticsearch_model):
+        return {
+            'odoo_id': self.id,
+            'elasticsearch_model': elasticsearch_model,
+            'backend_id': backend.id,
+        }
+
+    @api.multi
+    def get_binds(self, backend):
+        """
+
+        :return: list of the bind values
+        """
+        self.ensure_one()
+        return [self.bind_values(backend, self._binding_name or self._name)]
diff --git a/elasticsearch_connector_base/security/ir.model.access.csv b/elasticsearch_connector_base/security/ir.model.access.csv
@@ -0,0 +1,2 @@
+id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink
+access_elasticsearch_backend,access_elasticsearch_backend,model_elasticsearch_backend,,1,1,1,0
diff --git a/elasticsearch_connector_base/static/description/icon.png b/elasticsearch_connector_base/static/description/icon.png
