Merge branch 'master' into nirizr/basicblocks

Author: nirizr

.gitignore

@@ -96,3 +96,6 @@ server/db.sqlite3
 Thumbs.db
 *.un~
 *.swp
+
+# server private key file
+server/rematch/.rematch_secret.key

.travis.yml

@@ -3,14 +3,14 @@ language: python
 matrix:
   include:
     - python: "2.7"
-      env: PROJECT=idaplugin ENV=conda
+      env: PROJECT=idaplugin
     - python: "2.7"
-      env: PROJECT=server DJANGO_SETTINGS_MODULE='rematch.settings.debug' ENV=conda
+      env: PROJECT=server DJANGO_SETTINGS_MODULE='rematch.settings.debug'
       services:
         - mysql
         - rabbitmq
     - python: "3.5"
-      env: PROJECT=server DJANGO_SETTINGS_MODULE='rematch.settings.production' ENV=conda
+      env: PROJECT=server DJANGO_SETTINGS_MODULE='rematch.settings.production'
       services:
         - mysql
         - rabbitmq
@@ -19,6 +19,8 @@ matrix:
       services: docker
     - python: "2.7"
       env: PROJECT=setup.py
+    - python: "3.6"
+      env: PROJECT=setup.py
 
 branches:
   only:
@@ -34,47 +36,46 @@ before_install:
 
 install:
   - set +vx;
-  - if [ ${ENV} = "conda" ]; then
-      if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
-        wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh;
-      else
-        wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh;
-      fi;
-      bash ./miniconda.sh -b -p ${HOME}/miniconda;
-      export PATH=${HOME}/miniconda/bin:$PATH;
-      conda info -a;
-
-      if [ ${PROJECT} = "server" ]; then
-        conda install --yes python=${TRAVIS_PYTHON_VERSION} atlas numpy scipy scikit-learn;
-      fi;
-      if [ ${PROJECT} = "idaplugin" ]; then
-        conda install --yes python=${TRAVIS_PYTHON_VERSION} pyqt;
-      fi;
-      if [ -f tests/${PROJECT}/requirements.txt ]; then
-        pip install -r tests/${PROJECT}/requirements.txt ;
-      fi;
-      if [ -f ${PROJECT}/requirements.txt ]; then pip install -r ${PROJECT}/requirements.txt ; fi ;
-    fi;
+  - pip install --upgrade pip setuptools wheel
   - if [ ${ENV} = "docker" ]; then
       service docker start ;
       docker-compose -f ./${PROJECT}/docker-compose.yml build ;
       docker-compose -f ./${PROJECT}/docker-compose.yml up -d ;
       if [ -f ./tests/${PROJECT}/requirements.txt ]; then
         docker-compose -f ./${PROJECT}/docker-compose.yml exec web pip install -r ./tests/requirements.txt ;
       fi ;
-    fi;
-  - pip install flake8 coveralls pytest-cov ;
+    else
+      if [ ${PROJECT} = "idaplugin" ]; then
+        wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh;
+        bash ./miniconda.sh -b -p ${HOME}/miniconda;
+        export PATH=${HOME}/miniconda/bin:$PATH;
+        conda info -a;
+        conda install --yes python=${TRAVIS_PYTHON_VERSION} pyqt;
+      else
+        pip install --only-binary=numpy,scipy numpy scipy ;
+      fi ;
 
-before_script:
-  - if [ ${PROJECT} = "server" ]; then mysql -e 'SHOW VARIABLES LIKE "%version%";' ; fi ;
+      if [ -f tests/${PROJECT}/requirements.txt ]; then
+        pip install -r tests/${PROJECT}/requirements.txt ;
+      fi ;
+      if [ -f ${PROJECT}/requirements.txt ]; then
+        pip install -r ${PROJECT}/requirements.txt ;
+      fi ;
+    fi ;
+  - pip install flake8 coveralls pytest-cov ;
 
 script:
   - flake8 ./${PROJECT} --show-source --statistics
   - if [ -d ./tests/${PROJECT} ]; then flake8 ./tests/${PROJECT} --show-source ; fi ;
-  - if [ -d ./tests/${PROJECT} ] && [ ${ENV} = "conda" ]; then PYTHONPATH=. py.test ./${PROJECT} ./tests/${PROJECT} --verbose --cov-report= --cov=$PROJECT ; fi ;
-  - if [ -d ./tests/${PROJECT} ] && [ ${ENV} = "docker" ]; then docker-compose -f ./${PROJECT}/docker-compose.yml exec web py.test ./ --verbose ; fi ;
+  - if [ -d ./tests/${PROJECT} ]; then
+      if [ "${ENV}" = "docker" ]; then
+        docker-compose -f ./${PROJECT}/docker-compose.yml exec web py.test ./ --verbose ;
+      else
+        PYTHONPATH=. py.test ./${PROJECT} ./tests/${PROJECT} --verbose --cov-report= --cov=$PROJECT ;
+      fi ;
+    fi ;
+  - if [ ${PROJECT} = "setup.py" ]; then python ./setup.py server install ; fi ;
+  - if [ ${PROJECT} = "setup.py" ]; then python ./setup.py idaplugin install ; fi ;
 
 after_script:
-  - if [ -d ./tests/${PROJECT} ]; then
-      if [ ${ENV} != "docker" ]; then coveralls ; fi ;
-    fi;
+  - if [ -d ./tests/${PROJECT} ] && [ ${ENV} != "docker" ]; then coveralls ; echo "./tests/${PROJECT}" ; fi ;

README.rst

@@ -1,4 +1,4 @@
-|Read The Docs| |Build Status| |Code Health| |Codacy Badge| |idaplugin PyPI| |server PyPI|
+|Read The Docs| |Gitter Chat| |Build Status| |Code Health| |Codacy Badge| |idaplugin PyPI| |server PyPI|
 
 rematch
 =======
@@ -135,6 +135,9 @@ python scripting! :D).
 .. |Read The Docs| image:: https://readthedocs.org/projects/rematch/badge/?version=latest
    :alt: Read The Docs
    :target: http://rematch.readthedocs.io/en/latest/?badge=latest
+.. |Gitter Chat| image:: https://img.shields.io/gitter/room/rematch/rematch.js.svg
+   :alt: Gitter Chat
+   :target: https://gitter.im/rematch/rematch
 .. |Build Status| image:: https://travis-ci.org/nirizr/rematch.svg?branch=master
    :alt: Build Status
    :target: https://travis-ci.org/nirizr/rematch

idaplugin/rematch/actions/match.py

@@ -224,6 +224,7 @@ def accept_task(self):
 
   def start_results(self):
     self.pbar = QtWidgets.QProgressDialog()
+    self.pbar.setAutoReset(False)
     self.pbar.canceled.connect(self.cancel)
     self.pbar.rejected.connect(self.cancel)
     self.pbar.setLabelText("Receiving match results...")
@@ -290,6 +291,9 @@ def handle_page(self, response):
       self.pbar.setMaximum(self.pbar.maximum() + response['count'])
 
     new_value = max(self.pbar.value(), 0) + len(response['results'])
+    log('match_action').info("result download progress: {} / {} with {}"
+                             "".format(new_value, self.pbar.maximum(),
+                                       self.recieved))
     if new_value >= self.pbar.maximum() and len(self.recieved) >= 3:
       self.pbar.accept()
     else:

idaplugin/rematch/dialogs/widgets.py

@@ -1,6 +1,7 @@
 from ..idasix import QtWidgets, QtCore
 
 import ida_funcs
+import ida_kernwin
 import idc
 
 from .. import network
@@ -170,8 +171,8 @@ def label_clicked(self, checked):
 
   def btn_clicked(self, checked):
     del checked
-    f = ida_funcs.choose_func("Choose function to match with database",
-                              self.func.startEA if self.func else 0)
+    f = ida_kernwin.choose_func("Choose function to match with database",
+                                self.func.startEA if self.func else 0)
     if f:
       self.set_func(f)
       self.changed.emit()

idaplugin/rematch/idasix

@@ -1,19 +1,22 @@
 from rest_framework.decorators import api_view
 from rest_framework.response import Response
+from django.core.serializers.json import DjangoJSONEncoder
+import json
 
 
 @api_view(['GET'])
 def profile(request):
   user = {"id": request.user.id,
-          "is_authenticated": request.user.is_authenticated(),
+          "is_authenticated": request.user.is_authenticated,
           "is_active": request.user.is_active,
           "is_staff": request.user.is_staff,
           "is_superuser": request.user.is_superuser,
           }
-  if request.user.is_authenticated():
+  if request.user.is_authenticated:
     user.update({"username": request.user.username,
                  "first_name": request.user.first_name,
                  "last_name": request.user.last_name,
                  "email": request.user.email,
                  })
-  return Response(user)
+  response = json.dumps(user, cls=DjangoJSONEncoder)
+  return Response(response)

server/accounts/views.py

@@ -22,7 +22,8 @@ def match(cls, source, target):
     target_data = [json.loads(d) for d in target_data]
 
     dictvect = skl.feature_extraction.DictVectorizer()
-    source_matrix = dictvect.fit_transform(source_data)
+    source_matrix = dictvect.fit(source_data + target_data)
+    source_matrix = dictvect.transform(source_data)
     target_matrix = dictvect.transform(target_data)
     print("source matrix: {}, target matrix: {}".format(source_matrix.shape,
                                                         target_matrix.shape))

server/collab/matchers/dictionary_matcher.py

@@ -7,7 +7,7 @@
 
 class Project(models.Model):
   created = models.DateTimeField(auto_now_add=True)
-  owner = models.ForeignKey(User, db_index=True)
+  owner = models.ForeignKey(User, models.CASCADE, db_index=True)
   name = models.CharField(max_length=256)
   description = models.TextField()
   private = models.BooleanField()
@@ -22,8 +22,9 @@ class Meta(object):
 
 class File(models.Model):
   created = models.DateTimeField(auto_now_add=True)
-  owner = models.ForeignKey(User, db_index=True)
-  project = models.ForeignKey(Project, null=True, related_name='files')
+  owner = models.ForeignKey(User, models.CASCADE, db_index=True)
+  project = models.ForeignKey(Project, models.CASCADE, null=True,
+                              related_name='files')
   name = models.CharField(max_length=256)
   description = models.TextField()
   md5hash = models.CharField(max_length=32, db_index=True,
@@ -38,7 +39,7 @@ def __unicode__(self):
 
 class FileVersion(models.Model):
   created = models.DateTimeField(auto_now_add=True)
-  file = models.ForeignKey(File, related_name='versions')
+  file = models.ForeignKey(File, models.CASCADE, related_name='versions')
   md5hash = models.CharField(max_length=32,
                              validators=[MinLengthValidator(32)])
 
@@ -60,8 +61,9 @@ class Instance(models.Model):
                   (TYPE_EMPTY_FUNCTION, "Empty Function"),
                   (TYPE_FUNCTION, "Function"))
 
-  owner = models.ForeignKey(User, db_index=True)
-  file_version = models.ForeignKey(FileVersion, related_name='instances')
+  owner = models.ForeignKey(User, models.CASCADE, db_index=True)
+  file_version = models.ForeignKey(FileVersion, models.CASCADE,
+                                   related_name='instances')
   type = models.CharField(max_length=64, choices=TYPE_CHOICES)
   offset = models.BigIntegerField()
   size = models.BigIntegerField()
@@ -89,8 +91,10 @@ class Vector(models.Model):
                   (TYPE_MNEMONIC_HASH, "Mnemonic Hash"),
                   (TYPE_MNEMONIC_HIST, "Mnemonic Hist")]
 
-  instance = models.ForeignKey(Instance, related_name='vectors')
-  file_version = models.ForeignKey(FileVersion, related_name='vectors')
+  instance = models.ForeignKey(Instance, models.CASCADE,
+                               related_name='vectors')
+  file_version = models.ForeignKey(FileVersion, models.CASCADE,
+                                   related_name='vectors')
   type = models.CharField(max_length=64, choices=TYPE_CHOICES)
   type_version = models.IntegerField()
   data = models.TextField()
@@ -119,29 +123,32 @@ class Task(models.Model):
   created = models.DateTimeField(auto_now_add=True)
   finished = models.DateTimeField(null=True)
 
-  owner = models.ForeignKey(User, db_index=True)
+  owner = models.ForeignKey(User, models.CASCADE, db_index=True)
   status = models.CharField(default=STATUS_PENDING, max_length=64,
                             choices=STATUS_CHOICES)
 
-  source_file_version = models.ForeignKey(FileVersion,
+  source_file_version = models.ForeignKey(FileVersion, models.CASCADE,
                                           related_name='source_tasks')
   # TODO: make sure start > end
-  source_start = models.PositiveIntegerField(null=True)
-  source_end = models.PositiveIntegerField(null=True)
+  source_start = models.BigIntegerField(null=True)
+  source_end = models.BigIntegerField(null=True)
   # TODO: make sure only at least one of target_file/target_project is null
-  target_file = models.ForeignKey(File, null=True)
-  target_project = models.ForeignKey(Project, null=True)
+  target_file = models.ForeignKey(File, models.CASCADE, null=True)
+  target_project = models.ForeignKey(Project, models.CASCADE, null=True)
   matchers = models.TextField(default='[]')
 
   progress = models.PositiveSmallIntegerField(default=0)
   progress_max = models.PositiveSmallIntegerField(null=True)
 
 
 class Match(models.Model):
-  from_instance = models.ForeignKey(Instance, related_name='from_matches')
-  to_instance = models.ForeignKey(Instance, related_name='to_matches')
+  from_instance = models.ForeignKey(Instance, models.CASCADE,
+                                    related_name='from_matches')
+  to_instance = models.ForeignKey(Instance, models.CASCADE,
+                                  related_name='to_matches')
 
-  task = models.ForeignKey(Task, db_index=True, related_name='matches')
+  task = models.ForeignKey(Task, models.CASCADE, db_index=True,
+                           related_name='matches')
 
   type = models.CharField(max_length=64, choices=Vector.TYPE_CHOICES)
   score = models.FloatField()
@@ -155,6 +162,7 @@ class Annotation(models.Model):
                   (TYPE_ASSEMBLY, "Assembly"),
                   (TYPE_PROTOTYPE, "Prototype"))
 
-  instance = models.ForeignKey(Instance, related_name='annotations')
+  instance = models.ForeignKey(Instance, models.CASCADE,
+                               related_name='annotations')
   type = models.CharField(max_length=64, choices=TYPE_CHOICES)
   data = models.TextField()

server/collab/models.py

@@ -1,19 +1,15 @@
-from __future__ import absolute_import
-
+from __future__ import absolute_import, unicode_literals
 import os
-
 from celery import Celery
-import django
 
 # set the default Django settings module for the 'celery' program.
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'rematch.settings.production')
-django.setup()
 
-
-from django.conf import settings  # noqa
 app = Celery('rematch')
 
-# Using a string here means the worker will not have to
-# pickle the object when using Windows.
+# Using a string here means the worker doesn't have to serialize
+# the configuration object to child processes.
 app.config_from_object('django.conf:settings')
-app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
+
+# Load task modules from all registered Django app configs.
+app.autodiscover_tasks()