Merge branch 'develop' into 2.5

VERSION.json

@@ -1 +1 @@
-{"major":2, "minor":5, "hotfix":3}
+{"major":2, "minor":5, "hotfix":4}

app/Controller/AnalystDataController.php

@@ -194,6 +194,12 @@ public function view($type = 'Note', $id)
                 if (!$this->request->is('ajax')) {
                     unset($analystData[$this->modelSelection]['_canEdit']);
                 }
+                $children = $this->AnalystData->fetchChildNotesAndOpinions($this->Auth->user(), $analystData[$this->modelSelection], $this->_isRest(), 1);
+                foreach ($children as $child) {
+                    foreach ($child as $childType => $childData) {
+                        $analystData[$this->modelSelection][$childType][] = $childData;
+                    }
+                }
                 return $analystData;
             }
         ]);

app/Controller/AppController.php

@@ -34,7 +34,7 @@ class AppController extends Controller
     public $helpers = array('OrgImg', 'FontAwesome', 'UserName');
 
     private $__queryVersion = '168';
-    public $pyMispVersion = '2.5.3';
+    public $pyMispVersion = '2.5.4';
     public $phpmin = '8.1';
     public $phprec = '8.2';
     public $phptoonew = '9.0';
@@ -1368,8 +1368,7 @@ public function restSearch()
         $scope = empty($this->scopeOverride) ? $this->modelClass : $this->scopeOverride;
         if ($scope === 'MispObject') {
             $scope = 'Object';
-        }
-        if ($scope === 'MispAttribute') {
+        } else if ($scope === 'MispAttribute') {
             $scope = 'Attribute';
         }
         if (!isset($this->RestSearch->paramArray[$scope])) {
@@ -1379,7 +1378,7 @@ public function restSearch()
             $modelName = 'MispObject';
         } else if ($scope === 'Attribute') {
             $modelName = 'MispAttribute';
-        }else {
+        } else {
             $modelName = $scope;
         }
         if (!isset($this->$modelName)) {
@@ -1399,13 +1398,13 @@ public function restSearch()
         if (empty($filters) && $this->request->is('get')) {
             throw new BadRequestException(__('Restsearch queries using GET and no parameters are not allowed. If you have passed parameters via a JSON body, make sure you use POST requests.'));
         }
+        if ($filters === false) {
+            return $exception;
+        }
         if (empty($filters['returnFormat'])) {
             $filters['returnFormat'] = 'json';
         }
         unset($filterData);
-        if ($filters === false) {
-            return $exception;
-        }
 
         $user = $this->_closeSession();
 
@@ -1424,7 +1423,7 @@ public function restSearch()
         $responseType = empty($model->validFormats[$returnFormat][0]) ? 'json' : $model->validFormats[$returnFormat][0];
         // halt execution if we were to query for items above the ID. Blocks the endless caching bug
         if (!empty($filters['page']) && !empty($filters['returnFormat']) && $filters['returnFormat'] === 'cache') {
-            if ($this->__cachingOverflow($filters, $scope)) {
+            if ($this->__cachingOverflow($filters, $modelName, $scope)) {
                 $filename = $this->RestSearch->getFilename($filters, $scope, $responseType);
                 return $this->RestResponse->viewData('', $responseType, false, true, $filename, [
                     'X-Result-Count' => 0,
@@ -1455,13 +1454,14 @@ public function restSearch()
      * Halt execution if we were to query for items above the ID. Blocks the endless caching bug.
      *
      * @param array $filters
+     * @param string $modelName
      * @param string $scope
      * @return bool
      */
-    private function __cachingOverflow($filters, $scope)
+    private function __cachingOverflow(array $filters, $modelName, $scope)
     {
         $offset = ($filters['page'] * (empty($filters['limit']) ? 60 : $filters['limit'])) + 1;
-        $max_id = $this->$scope->query(sprintf('SELECT max(id) as max_id from %s;', Inflector::tableize($scope)));
+        $max_id = $this->$modelName->query(sprintf('SELECT max(id) as max_id from %s;', Inflector::tableize($scope)));
         $max_id = intval($max_id[0][0]['max_id']);
         if ($max_id < $offset) {
             return true;

app/Controller/AttributesController.php

@@ -2625,9 +2625,15 @@ public function addTag($id = false, $tag_id = false)
                                 if (empty($tagCollection)) {
                                     return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => 'Invalid Tag Collection.')), 'status'=>200, 'type' => 'json'));
                                 }
-                                $tag_id_list = array_column($tagCollection[0]['TagCollectionTag'], 'tag_id');
-                            } else {
+                                $tag_id_list = array_merge($tag_id_list, array_column($tagCollection[0]['TagCollectionTag'], 'tag_id'));
+                            } else if(is_numeric($tag_id)){
                                 $tag_id_list[] = $tag_id;
+                            } else {
+                                $tagId = $this->Attribute->AttributeTag->Tag->lookupTagIdForUser($this->Auth->user(), trim($tag_id));
+                                if (empty($tagId)) {
+                                    return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => 'Invalid Tag.')), 'status'=>200, 'type' => 'json'));
+                                }
+                                $tag_id_list[] = $tagId;
                             }
                         }
                     } else {

app/Controller/Component/ACLComponent.php

@@ -54,7 +54,7 @@ class ACLComponent extends Component
             'attributeStatistics' => array('*'),
             'bro' => array('*'),
             'checkAttachments' => array(),
-            'checkComposites' => array('perm_admin'),
+            'checkComposites' => array('perm_site_admin'),
             'checkOrphanedAttributes' => array(),
             'delete' => array('perm_add'),
             'deleteSelected' => array('perm_add'),

app/Controller/EventReportsController.php

@@ -485,10 +485,7 @@ public function importReportFromUrl($event_id)
             throw new MethodNotAllowedException(__('This function can only be reached via AJAX and via the API.'));
         }
         $errors = [];
-        $fetcherModule = $this->EventReport->isFetchURLModuleEnabledAndAllowed($this->Auth->user());
-        if (empty($fetcherModule)) {
-            $errors[] = __('Could not fetch report from URL. Fetcher module not enabled or could not download the page');
-        }
+        $fetcherModules = $this->EventReport->getEnabledFetchURLModules($this->Auth->user());
         if ($this->request->is('post')) {
             if (empty($this->data['EventReport'])) {
                 $this->data = ['EventReport' => $this->data];
@@ -507,7 +504,7 @@ public function importReportFromUrl($event_id)
             }
             $content = null;
             if (empty($errors)) {
-                $content = $this->EventReport->downloadMarkdownFromURL($event_id, $url, $format);
+                $content = $this->EventReport->downloadMarkdownFromURL($this->Auth->user(), $event_id, $url, $format);
                 if (!empty($content)) {
                     $report = [
                         'name' => __('Report from - %s (%s)', $url, time()),
@@ -529,7 +526,7 @@ public function importReportFromUrl($event_id)
                 return $this->__getSuccessResponseBasedOnContext($successMessage, $report, 'addFromURL', false, $redirectTarget);
             }
         }
-        $this->set('importModuleEnabled', is_array($fetcherModule));
+        $this->set('importModuleEnabled', !empty($fetcherModules));
         $this->set('event_id', $event_id);
         $this->layout = false;
         $this->render('ajax/importReportFromUrl');

app/Controller/EventsController.php

@@ -2354,7 +2354,12 @@ public function add_misp_export()
                     }
 
                     $ext = strtolower(pathinfo($file['name'], PATHINFO_EXTENSION));
-                    if (($ext !== 'xml' && $ext !== 'json') && $file['size'] > 0 && is_uploaded_file($file['tmp_name'])) {
+                    if (!is_uploaded_file($file['tmp_name']) || !($file['size'] > 0)) {
+                        $message = __('Something went wrong with the file upload.');
+                        $this->Flash->error($message);
+                        throw new MethodNotAllowedException($message);
+                    }
+                    if (($ext !== 'xml' && $ext !== 'json')) {
                         $log = ClassRegistry::init('Log');
                         $log->createLogEntry($this->Auth->user(), 'file_upload', 'Event', 0, 'MISP export file upload failed', 'File details: ' . json_encode($file));
                         $this->Flash->error(__('You may only upload MISP XML or MISP JSON files.'));
@@ -5201,7 +5206,6 @@ public function viewGalaxyMatrix($scope_id, $galaxy_id, $scope='event', $disable
         $this->set('galaxy_id', $galaxy_id);
         $this->set('eventId', $eventId);
         $this->set('extended', $extended);
-        $this->set('extending', $extending);
         $this->set('target_type', $scope);
         $this->set('columnOrders', $killChainOrders);
         $this->set('tabs', $tabs);

app/Controller/TemplateElementsController.php

@@ -50,6 +50,7 @@ public function index($id)
 
     public function templateElementAddChoices($id)
     {
+        $id = intval($id);
         if (!$this->_isSiteAdmin() && !$this->TemplateElement->Template->checkAuthorisation($id, $this->Auth->user(), true)) {
             throw new MethodNotAllowedException('You are not authorised to do that.');
         }
@@ -64,6 +65,7 @@ public function templateElementAddChoices($id)
     public function add($type, $id)
     {
         $ModelType = 'TemplateElement' . ucfirst($type);
+        $id = intval($id);
         if (!$this->_isSiteAdmin() && !$this->TemplateElement->Template->checkAuthorisation($id, $this->Auth->user(), true)) {
             return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => 'You are not authorised to do that.')), 'status' => 200, 'type' => 'json'));
         }
@@ -147,6 +149,7 @@ public function add($type, $id)
 
     public function edit($type, $id)
     {
+        $id = intval($id);
         $ModelType = 'TemplateElement' . ucfirst($type);
         $templateElement = $this->TemplateElement->find('first', array(
             'conditions' => array('TemplateElement.id' => $id),
@@ -227,6 +230,7 @@ public function edit($type, $id)
 
     public function delete($id)
     {
+        $id = intval($id);
         if (!$this->request->is('ajax')) {
             throw new MethodNotAllowedException('This action is for ajax requests only.');
         }

app/Controller/UsersController.php

@@ -2099,6 +2099,19 @@ private function __statisticsData(array $user, $params = array())
         $stats['object_count_month'] = $this->User->Event->Object->find('count', array('conditions' => array('Object.timestamp >' => $this_month, 'Object.deleted' => 0), 'recursive' => -1));
         $stats['objects_per_event'] = $stats['event_count'] != 0 ? round($stats['object_count'] / $stats['event_count']) : 0;
 
+        $this->loadModel('Note');
+        $this->loadModel('Opinion');
+        $this->loadModel('Relationship');
+        $stats['analyst_data_count'] = $this->Note->find('count', array('recursive' => -1)) +
+            $this->Opinion->find('count', array('recursive' => -1)) +
+            $this->Relationship->find('count', array('recursive' => -1));
+        $stats['analyst_data_count_month'] = $this->Note->find('count', array('conditions' => array('Note.modified >' => $this_month), 'recursive' => -1)) + 
+            $this->Opinion->find('count', array('conditions' => array('Opinion.modified >' => $this_month), 'recursive' => -1)) +
+            $this->Relationship->find('count', array('conditions' => array('Relationship.modified >' => $this_month), 'recursive' => -1));
+
+        $stats['eventreport_count'] = $this->User->Event->EventReport->find('count', array('conditions' => array('EventReport.deleted' => 0), 'recursive' => -1));
+        $stats['eventreport_count_month'] = $this->User->Event->EventReport->find('count', array('conditions' => array('EventReport.timestamp >' => $this_month, 'EventReport.deleted' => 0), 'recursive' => -1));
+
         $stats['correlation_count'] = $this->User->Event->Attribute->Correlation->find('count', array('recursive' => -1));
 
         $stats['proposal_count'] = $this->User->Event->ShadowAttribute->find('count', array('recursive' => -1, 'conditions' => array('deleted' => 0)));

app/Model/AnalystData.php

@@ -91,7 +91,7 @@ public function __construct($id = false, $table = null, $ds = null)
                 'Orgc' => [
                     'className' => 'Organisation',
                     'fields' => [
-                        'id', 'name', 'uuid','type', 'sector', 'nationality', 'local'
+                        'id', 'name', 'uuid','type', 'description', 'sector', 'nationality', 'local'
                     ],
                     'foreignKey' => false,
                     'conditions' => [
@@ -381,12 +381,12 @@ public function fetchSimple(array $user, $id): array
         ]);
     }
 
-    public function fetchChildNotesAndOpinions(array $user, array $analystData, $depth = 2): array
+    public function fetchChildNotesAndOpinions(array $user, array $analystData, $isRest = true, $depth = 2): array
     {
         if ($depth == 0 || !empty($this->fetchedUUIDFromRecursion[$analystData['uuid']])) {
             $hasMoreNotesOrOpinions =  $this->hasMoreNotesOrOpinions($analystData, $user);
             $analystData['_max_depth_reached'] = $hasMoreNotesOrOpinions;
-            return $analystData;
+            return $isRest ? [] : $analystData;
         }
         $this->fetchedUUIDFromRecursion[$analystData['uuid']] = true;
         $this->Note = ClassRegistry::init('Note');
@@ -415,13 +415,58 @@ public function fetchChildNotesAndOpinions(array $user, array $analystData, $dep
             ]
         ];
 
+        if ($isRest) {
+            // fetch notes and opinions as lists
+            $childNotesAndOpinions = [];
+            $childNotes = $this->Note->find('all', $paramsNote);
+            $childOpinions = $this->Opinion->find('all', $paramsOpinion);
+            $orgFields = ['id', 'uuid', 'name', 'type', 'description', 'sector', 'national', 'local'];
+            $orgTypes = ['Org', 'Orgc'];
+            if (!empty($childNotes)) {
+                foreach ($childNotes as $childNote) {
+                    foreach ($orgTypes as $orgType) {
+                        if (!empty($childOpinion['Note'][$orgType])) {
+                            $childNote['Note'][$orgType] = array_filter($childNote['Note'][$orgType], function ($key) use ($orgFields) {
+                                return in_array($key, $orgFields);
+                            }, ARRAY_FILTER_USE_KEY);    
+                        }
+                    }
+                    $childNotesAndOpinions[] = $childNote;
+                    $expandedNotesAndOpinions = $this->fetchChildNotesAndOpinions($user, $childNote['Note'], $isRest, $depth-1);
+                    if (!empty($expandedNotesAndOpinions)) {
+                        foreach ($expandedNotesAndOpinions as $expandedNoteOrOpinion) {
+                            $childNotesAndOpinions[] = $expandedNoteOrOpinion;
+                        }
+                    }
+                }
+            }
+            if (!empty($childOpinions)) {
+                foreach ($childOpinions as $childOpinion) {
+                    foreach ($orgTypes as $orgType) {
+                        if (!empty($childOpinion['Opinion'][$orgType])) {
+                            $childOpinion['Opinion'][$orgType] = array_filter($childOpinion['Opinion'][$orgType], function ($key) use ($orgFields) {
+                                return in_array($key, $orgFields);
+                            }, ARRAY_FILTER_USE_KEY);
+                        }
+                    }
+                    $childNotesAndOpinions[] = $childOpinion;
+                    $expandedNotesAndOpinions = $this->fetchChildNotesAndOpinions($user, $childOpinion['Opinion'], $isRest, $depth-1);
+                    if (!empty($expandedNotesAndOpinions)) {
+                        foreach ($expandedNotesAndOpinions as $expandedNoteOrOpinion) {
+                            $childNotesAndOpinions[] = $expandedNoteOrOpinion;
+                        }
+                    }
+                }
+            }
+            return $childNotesAndOpinions;
+        }
         // recursively fetch and include nested notes and opinions
-        $childNotes = array_map(function ($item) use ($user, $depth) {
-            $expandedNotes = $this->fetchChildNotesAndOpinions($user, $item['Note'], $depth-1);
+        $childNotes = array_map(function ($item) use ($user, $isRest, $depth) {
+            $expandedNotes = $this->fetchChildNotesAndOpinions($user, $item['Note'], $isRest, $depth-1);
             return $expandedNotes;
         }, $this->Note->find('all', $paramsNote));
-        $childOpinions = array_map(function ($item) use ($user, $depth) {
-            $expandedNotes = $this->fetchChildNotesAndOpinions($user, $item['Opinion'], $depth-1);
+        $childOpinions = array_map(function ($item) use ($user, $isRest, $depth) {
+            $expandedNotes = $this->fetchChildNotesAndOpinions($user, $item['Opinion'], $isRest, $depth-1);
             return $expandedNotes;
         }, $this->Opinion->find('all', $paramsOpinion));
 
@@ -432,8 +477,8 @@ public function fetchChildNotesAndOpinions(array $user, array $analystData, $dep
             $analystData['Note'] = $childNotes;
         }
         if (!empty($childOpinions)) {
-            foreach ($childNotes as $childNote) {
-                $this->fetchedUUIDFromRecursion[$childNote['uuid']] = true;
+            foreach ($childOpinions as $childOpinion) {
+                $this->fetchedUUIDFromRecursion[$childOpinion['uuid']] = true;
             }
             $analystData['Opinion'] = $childOpinions;
         }
@@ -496,7 +541,7 @@ public function getChildren($user, $uuid, $depth=2): array
         $analystData = $analystData[$this->alias];
         $this->Note = ClassRegistry::init('Note');
         $this->Opinion = ClassRegistry::init('Opinion');
-        $analystData = $this->fetchChildNotesAndOpinions($user, $analystData, $depth);
+        $analystData = $this->fetchChildNotesAndOpinions($user, $analystData, isset($this->__isRest) ? $this->__isRest : false, $depth);
         return $analystData;
     }