Skip to content

Commit 4cdcb3e

Browse files
etobellaetobella
committed
First version
1 parent 5706d3e commit 4cdcb3e

File tree

15 files changed

+382
-0
lines changed

15 files changed

+382
-0
lines changed

‎dependencies.txt

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
elasticsearch

‎elasticsearch_connector_base/__init__.py

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# Copyright 2017 Creu Blanca
2+
# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html).
3+
4+
from . import components
5+
from . import models

‎elasticsearch_connector_base/__manifest__.py

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
# Copyright 2017 Creu Blanca
2+
# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html).
3+
4+
{
5+
'name': 'Elasticsearch Connector Base',
6+
'version': '11.0.1.0.1',
7+
'category': 'Elasticsearch connector',
8+
'depends': [
9+
'connector',
10+
],
11+
'author': 'Creu Blanca',
12+
'license': 'AGPL-3',
13+
'summary': 'Elasticsearch connector base',
14+
'data': [
15+
'data/backend.xml',
16+
'security/ir.model.access.csv',
17+
],
18+
'external_dependencies': {
19+
'python': [
20+
'elasticsearch'
21+
],
22+
},
23+
'installable': True,
24+
}

‎elasticsearch_connector_base/components/__init__.py

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# Copyright 2017 Creu Blanca
2+
# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html).
3+
4+
from . import core
5+
from . import binder
6+
from . import exporter
7+
from . import listener

‎elasticsearch_connector_base/components/binder.py

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
# Copyright 2017 Creu Blanca
2+
# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html).
3+
4+
from odoo.addons.component.core import Component
5+
6+
7+
class ElasticsearchModelBinder(Component):
8+
_name = 'elasticsearch.binder'
9+
_inherit = ['base.binder', 'base.elasticsearch.connector']
10+
_apply_on = False

‎elasticsearch_connector_base/components/core.py

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# Copyright 2017 Creu Blanca
2+
# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html).
3+
4+
from odoo.addons.component.core import AbstractComponent
5+
6+
7+
class BaseElasticsearchConnectorComponent(AbstractComponent):
8+
""" Base elasticsearch Connector Component
9+
All components of this connector should inherit from it.
10+
"""
11+
12+
_name = 'base.elasticsearch.connector'
13+
_inherit = 'base.connector'
14+
_collection = 'elasticsearch.backend'

‎elasticsearch_connector_base/components/exporter.py

+141
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
# Copyright 2017 Creu Blanca
2+
# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html).
3+
4+
import logging
5+
import psycopg2
6+
import simplejson
7+
import requests
8+
from datetime import datetime
9+
10+
import odoo
11+
from odoo.addons.component.core import AbstractComponent
12+
from odoo.addons.connector.exception import RetryableJobError
13+
import elasticsearch
14+
15+
ISO_FORMAT = '%Y-%m-%dT%H:%M:%S.%f'
16+
_logger = logging.getLogger(__name__)
17+
18+
19+
class ElasticsearchBaseExporter(AbstractComponent):
20+
""" Base exporter for the Elasticsearch """
21+
22+
_name = 'elasticsearch.base.exporter'
23+
_inherit = ['base.exporter', 'base.elasticsearch.connector']
24+
_usage = 'record.exporter'
25+
_exporter_failure_timeout = 2
26+
27+
def _lock(self, binding):
28+
""" Lock the binding record.
29+
Lock the binding record so we are sure that only one export
30+
job is running for this record if concurrent jobs have to export the
31+
same record.
32+
When concurrent jobs try to export the same record, the first one
33+
will lock and proceed, the others will fail to lock and will be
34+
retried later.
35+
This behavior works also when the export becomes multilevel
36+
with :meth:`_export_dependencies`. Each level will set its own lock
37+
on the binding record it has to export.
38+
"""
39+
sql = ("SELECT id FROM %s WHERE ID = %%s FOR UPDATE NOWAIT" %
40+
self.model._table)
41+
try:
42+
self.env.cr.execute(sql, (binding.id,),
43+
log_exceptions=False)
44+
except psycopg2.OperationalError:
45+
_logger.info('A concurrent job is already exporting the same '
46+
'record (%s with id %s). Job delayed later.',
47+
self.model._name, binding.id)
48+
raise RetryableJobError(
49+
'A concurrent job is already exporting the same record '
50+
'(%s with id %s). The job will be retried later.' %
51+
(self.model._name, binding.id),
52+
seconds=self._exporter_failure_timeout)
53+
54+
def create(self, binding, *args, **kwargs):
55+
self._lock(binding)
56+
index = binding.index
57+
doc_type = binding.doc_type
58+
es = elasticsearch.Elasticsearch(
59+
hosts=binding.backend_id.get_hosts())
60+
data = simplejson.dumps(kwargs['data'])
61+
es.create(index, doc_type, binding.id, body=data)
62+
binding.sync_date = kwargs['sync_date']
63+
if not odoo.tools.config['test_enable']:
64+
self.env.cr.commit() # noqa
65+
self._after_export()
66+
return True
67+
68+
def delete(self, binding, *args, **kwargs):
69+
""" Run the synchronization
70+
:param binding: binding record to export
71+
"""
72+
self._lock(binding)
73+
sync_date = datetime.strptime(kwargs['sync_date'], ISO_FORMAT)
74+
if (
75+
not binding.sync_date or
76+
sync_date >= datetime.strptime(binding.sync_date, ISO_FORMAT)
77+
):
78+
index = binding.index
79+
doc_type = binding.doc_type
80+
es = elasticsearch.Elasticsearch(
81+
hosts=binding.backend_id.get_hosts())
82+
es.delete(index, doc_type, binding.id)
83+
binding.sync_date = kwargs['sync_date']
84+
if not odoo.tools.config['test_enable']:
85+
self.env.cr.commit() # noqa
86+
binding.sync_date = kwargs['sync_date']
87+
else:
88+
_logger.info(
89+
'Record from %s with id %s has already been sended (%s), so it'
90+
' is deprecated ' % (
91+
self.model._name, binding.id, kwargs['sync_date']
92+
)
93+
)
94+
# Commit so we keep the external ID when there are several
95+
# exports (due to dependencies) and one of them fails.
96+
# The commit will also release the lock acquired on the binding
97+
# record
98+
if not odoo.tools.config['test_enable']:
99+
self.env.cr.commit() # noqa
100+
self._after_export()
101+
return True
102+
103+
def update(self, binding, *args, **kwargs):
104+
""" Run the synchronization
105+
:param binding: binding record to export
106+
"""
107+
self._lock(binding)
108+
sync_date = datetime.strptime(kwargs['sync_date'], ISO_FORMAT)
109+
if (
110+
not binding.sync_date or
111+
sync_date >= datetime.strptime(binding.sync_date, ISO_FORMAT)
112+
):
113+
index = binding.index
114+
doc_type = binding.doc_type
115+
es = elasticsearch.Elasticsearch(
116+
hosts=binding.backend_id.get_hosts())
117+
data = simplejson.dumps(kwargs['data'])
118+
es.update(index, doc_type, binding.id, body=data)
119+
binding.sync_date = kwargs['sync_date']
120+
if not odoo.tools.config['test_enable']:
121+
self.env.cr.commit() # noqa
122+
binding.sync_date = kwargs['sync_date']
123+
else:
124+
_logger.info(
125+
'Record from %s with id %s has already been sended (%s), so it'
126+
' is deprecated ' % (
127+
self.model._name, binding.id, kwargs['sync_date']
128+
)
129+
)
130+
# Commit so we keep the external ID when there are several
131+
# exports (due to dependencies) and one of them fails.
132+
# The commit will also release the lock acquired on the binding
133+
# record
134+
if not odoo.tools.config['test_enable']:
135+
self.env.cr.commit() # noqa
136+
self._after_export()
137+
return True
138+
139+
def _after_export(self):
140+
""" Can do several actions after exporting a record"""
141+
pass

‎elasticsearch_connector_base/components/listener.py

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# Copyright 2017 Creu Blanca
2+
# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html).
3+
4+
from odoo.addons.component.core import AbstractComponent
5+
from datetime import datetime
6+
7+
8+
class AbstractElasticsearchListener(AbstractComponent):
9+
_name = 'abstract.elasticsearch.listener'
10+
_inherit = 'base.event.listener'
11+
_apply_on = False
12+
13+
def on_record_create(self, record):
14+
for rec in record:
15+
for backend in rec.get_backends():
16+
for vals in rec.get_binds(backend):
17+
binding = self.env[rec.get_binding_model()].create(vals)
18+
binding.with_delay().export_create(
19+
datetime.now().isoformat())
20+
21+
def on_record_write(self, record):
22+
for rec in record:
23+
if rec._fields.get('elasticsearch_bind_ids'):
24+
for binding in rec.elasticsearch_bind_ids:
25+
binding.with_delay().export_update(
26+
datetime.now().isoformat())
27+
28+
def on_record_unlink(self, record):
29+
for rec in record:
30+
for binding in rec.elasticsearch_bind_ids:
31+
binding.with_delay().binding.export_delete(
32+
datetime.now().isoformat())

‎elasticsearch_connector_base/data/backend.xml

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
<?xml version="1.0" encoding="UTF-8" ?>
2+
<odoo>
3+
<record id="elasticsearch" model="elasticsearch.backend">
4+
<field name="name">Elasticsearch</field>
5+
</record>
6+
</odoo>

‎elasticsearch_connector_base/models/__init__.py

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# Copyright 2017 Creu Blanca
2+
# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html).
3+
4+
from . import backend
5+
from . import binding
6+
from . import elasticsearch_model

‎elasticsearch_connector_base/models/backend.py

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
# Copyright 2017 Creu Blanca
2+
# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html).
3+
4+
from odoo import api, fields, models
5+
from odoo.addons.server_environment import serv_config
6+
7+
BUS_SECTION = 'elasticsearch'
8+
9+
10+
class ElasticsearchBackend(models.Model):
11+
_name = 'elasticsearch.backend'
12+
_description = 'Elasticsearch Backend'
13+
_inherit = 'connector.backend'
14+
15+
name = fields.Char(
16+
readonly=True
17+
)
18+
destination = fields.Char(
19+
string='Destination',
20+
)
21+
port = fields.Integer()
22+
23+
def get_hosts(self):
24+
return [{"host": self.destination, "port": self.port}]

‎elasticsearch_connector_base/models/binding.py

+71
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
# Copyright 2017 Creu Blanca
2+
# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html).
3+
4+
from datetime import datetime
5+
6+
from odoo import api, models, fields
7+
from odoo.addons.queue_job.job import job
8+
9+
10+
class ElasticsearchBinding(models.AbstractModel):
11+
_name = 'elasticsearch.binding'
12+
_inherit = 'external.binding'
13+
_description = 'Elasticsearch abstract Binding'
14+
_elasticsearch_fields = []
15+
16+
odoo_id = fields.Many2one(
17+
required=True,
18+
comodel_name='elasticsearch.model',
19+
)
20+
index = fields.Char()
21+
doc_type = fields.Char()
22+
backend_id = fields.Many2one(
23+
comodel_name='elasticsearch.backend',
24+
string='CB Backend',
25+
required=True,
26+
ondelete='restrict',
27+
)
28+
elasticsearch_document_id = fields.Integer()
29+
sync_date = fields.Char()
30+
31+
@job(default_channel='root.elasticsearch')
32+
@api.multi
33+
def export_create(self, date=datetime.now().isoformat()):
34+
self.ensure_one()
35+
with self.backend_id.work_on(self._name) as work:
36+
exporter = work.component(usage='record.exporter')
37+
return exporter.create(
38+
self,
39+
data=self.get_binding_values(),
40+
sync_date=date)
41+
42+
@job(default_channel='root.elasticsearch')
43+
@api.multi
44+
def export_update(self, date=datetime.now().isoformat()):
45+
self.ensure_one()
46+
with self.backend_id.work_on(self._name) as work:
47+
exporter = work.component(usage='record.exporter')
48+
return exporter.update(
49+
self,
50+
data=self.get_binding_values(),
51+
sync_date=date)
52+
53+
@job(default_channel='root.elasticsearch')
54+
@api.multi
55+
def export_delete(self, date=datetime.now().isoformat()):
56+
self.ensure_one()
57+
with self.backend_id.work_on(self._name) as work:
58+
exporter = work.component(usage='record.exporter')
59+
return exporter.delete(
60+
self,
61+
document_id=self.elasticsearch_document_id,
62+
sync_date=date
63+
)
64+
65+
@api.model
66+
def get_fields(self):
67+
return ['id', 'write_date', 'write_uid', 'create_date', 'create_uid']
68+
69+
def get_binding_values(self):
70+
flds = list(set(self._elasticsearch_fields + self.get_fields()))
71+
return self.odoo_id.read(flds, load=False)[0]

‎elasticsearch_connector_base/models/bus_model.py

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
# Copyright 2017 Creu Blanca
2+
# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl.html).
3+
4+
from odoo import api, fields, models
5+
6+
7+
class ElasticsearchModel(models.AbstractModel):
8+
_name = 'elasticsearch.model'
9+
_binding_name = False
10+
11+
elasticsearch_bind_ids = fields.One2many(
12+
comodel_name='elasticsearch.binding',
13+
inverse_name='odoo_id',
14+
string='Elasticsearch Bindings'
15+
)
16+
17+
@api.model
18+
def get_backends(self):
19+
return self.env['elasticsearch.backend'].search([])
20+
21+
@api.model
22+
def get_binding_model(self):
23+
return 'elasticsearch.binding.' + self._name
24+
25+
def bind_values(self, backend, elasticsearch_model):
26+
return {
27+
'odoo_id': self.id,
28+
'elasticsearch_model': elasticsearch_model,
29+
'backend_id': backend.id,
30+
}
31+
32+
@api.multi
33+
def get_binds(self, backend):
34+
"""
35+
36+
:return: list of the bind values
37+
"""
38+
self.ensure_one()
39+
return [self.bind_values(backend, self._binding_name or self._name)]

‎elasticsearch_connector_base/security/ir.model.access.csv

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink
2+
access_elasticsearch_backend,access_elasticsearch_backend,model_elasticsearch_backend,,1,1,1,0

‎elasticsearch_connector_base/static/description/icon.png

6.42 KB
Loading

0 commit comments

Comments
 (0)