Merge branch 'main' into unskip-waf-blocking-tests-for-ruby

.gitlab-ci.yml

@@ -68,7 +68,7 @@ onboarding_nodejs:
         - ONBOARDING_FILTER_WEBLOG: [test-app-nodejs,test-app-nodejs-container]
           SCENARIO: [INSTALLER_AUTO_INJECTION,SIMPLE_AUTO_INJECTION_PROFILING]
           DEFAULT_VMS: ["True", "False"]
-        - ONBOARDING_FILTER_WEBLOG: [test-app-nodejs-16, test-app-nodejs-unsupported-defaults]
+        - ONBOARDING_FILTER_WEBLOG: [test-app-nodejs-08, test-app-nodejs-16, test-app-nodejs-unsupported-defaults]
           SCENARIO: [INSTALLER_NOT_SUPPORTED_AUTO_INJECTION]
           DEFAULT_VMS: ["True", "False"]
         - ONBOARDING_FILTER_WEBLOG: [test-app-nodejs]

docs/edit/skip-tests.md

@@ -1,9 +1,9 @@
 Three decorators allow you to skip test functions or classes for a library:
 
 * `@irrelevant`: The tested feature/behavior is irrelevant to the library, meaning the feature is either purposefully not supported by the lib or cannot reasonably be implemented
-* `@bug`: The lib does not implement the feature correctly/up to spec
+* `@missing_feature`: The tested feature/behavior does not exist in the library or there is a deficit in the test library that blocks this test from executing for the lib. **The test will be executed** and being ignored if it fails. If it passes, a warning will be added in thee output (`XPASS`)
+* `@bug`: The lib does not implement the feature correctly/up to spec. **The test will be executed** and being ignored if it fails. If it passes, a warning will be added in thee output (`XPASS`)
 * `@flaky` (subclass of `bug`): The feature sometimes fails, sometimes passes. It's not reliable, so don't run it.
-* `@missing_feature`: The tested feature/behavior does not exist in the library or there is a deficit in the test library that blocks this test from executing for the lib
 
 To skip specific test functions within a test class, use them as in-line decorators (Example below).
 To skip test classes or test files, use the decorator in the library's [manifest file](./manifest.md).
@@ -14,6 +14,7 @@ The decorators take several arguments:
 * `library`: provide library. version numbers are allowed e.g.`[email protected]`, see [versions.md](./versions.md) for more details on semantic versioning and testing against unmerged changes
 * `weblog_variant`: if you want to skip the test for a specific weblog
 * `reason`: why the test is skipped. It's especially useful for `@bug`, in which case the value should reference a JIRA ticket number.
+* `force_skip`: if you want to not execute a test maked with `missing_feature` or `bug` (main reason it entirely break the app), you can set `force_skip` to `True`
 
 
 ```python

lib-injection/build/docker/nodejs/sample-app/child.js

@@ -1,6 +1,6 @@
 console.log('Child process started');
 
-setTimeout(() => {
+setTimeout(function () {
   console.log('Child process exiting after 5 seconds');
   process.kill(process.pid, 'SIGSEGV');
 }, 5000); // Add a delay before crashing otherwise the telemetry forwarder leaves a zombie behind

lib-injection/build/docker/nodejs/sample-app/index.js

@@ -1,50 +1,52 @@
-const fs = require('fs');
-const path = require('path');
-const { fork } = require('child_process');
+var fs = require('fs');
+var path = require('path');
+var fork = require('child_process').fork;
 
-process.on('SIGTERM', (signal) => {
+process.on('SIGTERM', function (signal) {
   process.exit(0);
 });
 
 function crashme(req, res) {
-  setTimeout(() => {
+  setTimeout(function () {
     process.kill(process.pid, 'SIGSEGV');
 
     res.writeHead(200, { 'Content-Type': 'text/plain' });
-    res.end(`Crashing process ${process.pid}`);
+    res.end('Crashing process ' + process.pid);
   }, 2000); // Add a delay before crashing otherwise the telemetry forwarder leaves a zombie behind
 }
 
 function forkAndCrash(req, res) {
-  const child = fork('child.js');
+  var child = fork('child.js');
 
-  child.on('close', (code, signal) => {
+  child.on('close', function (code, signal) {
     res.writeHead(200, { 'Content-Type': 'text/plain' });
-    res.end(`Child process ${child.pid} exited with code ${code}, signal ${signal}`);
+    res.end('Child process ' + child.pid + ' exited with code ' + code + ', signal ' + signal);
   });
 }
 
 function getChildPids(req, res) {
-  const currentPid = process.pid;
+  var currentPid = process.pid;
 
   try {
     // Get the list of all process directories in /proc
-    const procDir = '/proc';
-    const procFiles = fs.readdirSync(procDir).filter(file => /^\d+$/.test(file));
+    var procDir = '/proc';
+    var procFiles = fs.readdirSync(procDir).filter(function (file) {
+      return /^\d+$/.test(file)
+    });
 
-    let childPids = [];
+    var childPids = [];
 
     // Iterate through each process directory
-    procFiles.forEach(pid => {
-      const statusPath = path.join(procDir, pid, 'status');
+    procFiles.forEach(function (pid) {
+      var statusPath = path.join(procDir, pid, 'status');
       try {
         if (fs.existsSync(statusPath)) {
-          const statusContent = fs.readFileSync(statusPath, 'utf8');
+          var statusContent = fs.readFileSync(statusPath, 'utf8');
 
           // Find the PPid line in the status file
-          const ppidMatch = statusContent.match(/^PPid:\s+(\d+)/m);
+          var ppidMatch = statusContent.match(/^PPid:\s+(\d+)/m);
           if (ppidMatch) {
-            const ppid = parseInt(ppidMatch[1], 10);
+            var ppid = parseInt(ppidMatch[1], 10);
             if (ppid === currentPid) {
               childPids.push(pid);
             }
@@ -60,39 +62,41 @@ function getChildPids(req, res) {
 
     // Send response back
     res.writeHead(200, { 'Content-Type': 'text/plain' });
-    res.end(`${childPids.join(', ')}`);
+    res.end(childPids.join(', '));
   } catch (error) {
     res.writeHead(500, { 'Content-Type': 'text/plain' });
-    res.end(`Error: ${error.message}`);
+    res.end('Error: ' + error.message);
   }
 }
 
 function getZombies(req, res) {
   try {
-    const procDir = '/proc';
-    const procFiles = fs.readdirSync(procDir).filter(file => /^\d+$/.test(file));
-    let zombieProcesses = [];
+    var procDir = '/proc';
+    var procFiles = fs.readdirSync(procDir).filter(function (file) {
+      return /^\d+$/.test(file)
+    });
+    var zombieProcesses = [];
 
     // Iterate through each process directory
-    procFiles.forEach(pid => {
-      const statusPath = path.join(procDir, pid, 'status');
+    procFiles.forEach(function (pid) {
+      var statusPath = path.join(procDir, pid, 'status');
       try {
         if (fs.existsSync(statusPath)) {
-          const statusContent = fs.readFileSync(statusPath, 'utf8');
+          var statusContent = fs.readFileSync(statusPath, 'utf8');
 
           // Find the Name, State, and PPid lines in the status file
-          const nameMatch = statusContent.match(/^Name:\s+(\S+)/m);
-          const stateMatch = statusContent.match(/^State:\s+(\w)/m);
-          const ppidMatch = statusContent.match(/^PPid:\s+(\d+)/m);
+          var nameMatch = statusContent.match(/^Name:\s+(\S+)/m);
+          var stateMatch = statusContent.match(/^State:\s+(\w)/m);
+          var ppidMatch = statusContent.match(/^PPid:\s+(\d+)/m);
 
           if (nameMatch && stateMatch && ppidMatch) {
-            const name = nameMatch[1];
-            const state = stateMatch[1];
-            const ppid = ppidMatch[1];
+            var name = nameMatch[1];
+            var state = stateMatch[1];
+            var ppid = ppidMatch[1];
 
             // Check if the process state is 'Z' (zombie)
             if (state === 'Z') {
-              zombieProcesses.push(`${name} (PID: ${pid}, PPID: ${ppid})`);
+              zombieProcesses.push(name + ' (PID: ' + pid + ', PPID: ' + ppid + ')');
             }
           }
         }
@@ -106,14 +110,14 @@ function getZombies(req, res) {
 
     // Send response back
     res.writeHead(200, { 'Content-Type': 'text/plain' });
-    res.end(`${zombieProcesses.join(', ')}`);
+    res.end(zombieProcesses.join(', '));
   } catch (error) {
     res.writeHead(500, { 'Content-Type': 'text/plain' });
-    res.end(`Error: ${error.message}`);
+    res.end('Error: ' + error.message);
   }
 }
 
-require('http').createServer((req, res) => {
+require('http').createServer(function (req, res) {
   if (req.url === '/crashme') {
     crashme(req, res);
   } else if (req.url === '/fork_and_crash') {
@@ -125,6 +129,6 @@ require('http').createServer((req, res) => {
   } else {
     res.end('Hello, world!\n')
   }
-}).listen(18080, () => {
+}).listen(18080, function () {
   console.log('listening on port 18080') // eslint-disable-line no-console
 })

tests/test_the_test/test_force_skip.py

@@ -0,0 +1,42 @@
+from utils import bug, missing_feature, scenarios
+
+from .utils import run_system_tests
+
+FILENAME = "tests/test_the_test/test_force_skip.py"
+
+
+@scenarios.test_the_test
+class Test_ForceSkip:
+    def test_force_bug(self):
+        tests = run_system_tests(test_path=FILENAME)
+
+        nodeid = f"{FILENAME}::Test_Bug::test_bug_executed"
+        assert tests[nodeid]["outcome"] == "xpassed"
+
+        nodeid = f"{FILENAME}::Test_Bug::test_missing_feature_executed"
+        assert tests[nodeid]["outcome"] == "xpassed"
+
+        nodeid = f"{FILENAME}::Test_Bug::test_bug_not_executed"
+        assert tests[nodeid]["outcome"] == "skipped"
+
+        nodeid = f"{FILENAME}::Test_Bug::test_missing_feature_not_executed"
+        assert tests[nodeid]["outcome"] == "skipped"
+
+
+@scenarios.mock_the_test
+class Test_Bug:
+    @bug(True)
+    def test_bug_executed(self):
+        assert True
+
+    @missing_feature(True)
+    def test_missing_feature_executed(self):
+        assert True
+
+    @bug(True, force_skip=True)
+    def test_bug_not_executed(self):
+        assert True
+
+    @missing_feature(True, force_skip=True)
+    def test_missing_feature_not_executed(self):
+        assert True

utils/_decorators.py

@@ -70,12 +70,15 @@ def _get_skipped_item(item, skip_reason):
     return item
 
 
-def _get_expected_failure_item(item, skip_reason):
+def _get_expected_failure_item(item, skip_reason, force_skip: bool = False):
     if inspect.isfunction(item) or inspect.isclass(item):
         if not hasattr(item, "pytestmark"):
             setattr(item, "pytestmark", [])
 
-        item.pytestmark.append(pytest.mark.xfail(reason=skip_reason))
+        if force_skip:
+            item.pytestmark.append(pytest.mark.skip(reason=skip_reason))
+        else:
+            item.pytestmark.append(pytest.mark.xfail(reason=skip_reason))
     else:
         raise ValueError(f"Unexpected skipped object: {item}")
 
@@ -111,7 +114,7 @@ def _should_skip(condition=None, library=None, weblog_variant=None):
     return True
 
 
-def missing_feature(condition: bool = None, library=None, weblog_variant=None, reason=None):
+def missing_feature(condition: bool = None, library=None, weblog_variant=None, reason=None, force_skip: bool = False):
     """decorator, allow to mark a test function/class as missing"""
 
     skip = _should_skip(library=library, weblog_variant=weblog_variant, condition=condition)
@@ -126,7 +129,7 @@ def decorator(function_or_class):
 
         full_reason = "missing_feature" if reason is None else f"missing_feature ({reason})"
 
-        return _get_expected_failure_item(function_or_class, full_reason)
+        return _get_expected_failure_item(function_or_class, full_reason, force_skip=force_skip)
 
     return decorator
 
@@ -150,7 +153,7 @@ def decorator(function_or_class):
     return decorator
 
 
-def bug(condition=None, library=None, weblog_variant=None, reason=None):
+def bug(condition=None, library=None, weblog_variant=None, reason=None, force_skip: bool = False):
     """
     Decorator, allow to mark a test function/class as an known bug.
     The test is executed, and if it passes, and warning is reported
@@ -169,7 +172,7 @@ def decorator(function_or_class):
             return function_or_class
 
         full_reason = "bug" if reason is None else f"bug ({reason})"
-        return _get_expected_failure_item(function_or_class, full_reason)
+        return _get_expected_failure_item(function_or_class, full_reason, force_skip=force_skip)
 
     return decorator
 

utils/build/virtual_machine/weblogs/nodejs/provision_test-app-nodejs-08.yml

@@ -0,0 +1,34 @@
+lang_variant:
+    name: node08
+    version: 0.8
+    cache: true
+    install: 
+      - os_type: linux
+        remote-command: |
+          curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash
+          export NVM_DIR="$HOME/.nvm"
+          [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"  # This loads nvm
+          [ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion"  # This loads nvm bash_completion
+          nvm install --no-progress 0.8
+          nvm use node
+          n=$(which node);n=${n%/bin/node}; chmod -R 755 $n/bin/*; sudo cp -r $n/{bin,lib,share} /usr/local
+
+weblog:
+    name: test-app-nodejs-08
+    exact_os_branches: [ubuntu20_amd64, centos_7_amd64]
+    install: 
+      - os_type: linux
+        copy_files:
+          - name: copy-service
+            local_path: utils/build/virtual_machine/weblogs/common/test-app.service
+
+          - name: copy-service-run-script
+            local_path: utils/build/virtual_machine/weblogs/common/create_and_run_app_service.sh
+
+          - name: copy-run-weblog-script
+            local_path: utils/build/virtual_machine/weblogs/nodejs/test-app-nodejs/test-app-nodejs_run.sh
+
+          - name: copy-binary
+            local_path: lib-injection/build/docker/nodejs/sample-app/index.js
+
+        remote-command: sh test-app-nodejs_run.sh