Skip to content

Commit afb824d

Browse files
albertoabellagarciaalbertoabellagarcia
committed
new testing method
1 parent 0022351 commit afb824d

8 files changed

+850
-0
lines changed

‎test_data_model/master_tests.py

+214
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,214 @@
1+
#################################################################################
2+
# Licensed to the FIWARE Foundation (FF) under one #
3+
# or more contributor license agreements. The FF licenses this file #
4+
# to you under the Apache License, Version 2.0 (the "License") #
5+
# you may not use this file except in compliance with the License. #
6+
# You may obtain a copy of the License at #
7+
# #
8+
# http://www.apache.org/licenses/LICENSE-2.0 #
9+
# #
10+
# Unless required by applicable law or agreed to in writing, software #
11+
# distributed under the License is distributed on an "AS IS" BASIS, #
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
13+
# See the License for the specific language governing permissions and #
14+
# limitations under the License. #
15+
# Author: Alberto Abella #
16+
#################################################################################
17+
18+
import json
19+
import importlib
20+
import sys
21+
import os
22+
import requests
23+
import shutil
24+
25+
def is_url(path):
26+
"""
27+
Check if the provided path is a URL.
28+
29+
Parameters:
30+
path (str): The path to check.
31+
32+
Returns:
33+
bool: True if the path is a URL, False otherwise.
34+
"""
35+
return path.startswith("http://") or path.startswith("https://")
36+
37+
def convert_github_url_to_raw(repo_url):
38+
"""
39+
Convert a GitHub repository URL to the corresponding raw file URL.
40+
41+
Parameters:
42+
repo_url (str): The GitHub repository URL (e.g., https://github.com/smart-data-models/dataModel.Weather/blob/master/WeatherObserved/schema.json).
43+
44+
Returns:
45+
str: The raw file base URL (e.g., https://raw.githubusercontent.com/smart-data-models/dataModel.Weather/refs/heads/master/WeatherObserved/).
46+
"""
47+
try:
48+
if "github.com" not in repo_url:
49+
raise ValueError("Invalid GitHub repository URL.")
50+
51+
# Handle master branch URLs
52+
if "/blob/master/" in repo_url:
53+
# Replace "github.com" with "raw.githubusercontent.com"
54+
raw_url = repo_url.replace("github.com", "raw.githubusercontent.com")
55+
# Replace "/blob/master/" with "/refs/heads/master/"
56+
raw_url = raw_url.replace("/blob/master/", "/refs/heads/master/")
57+
return raw_url
58+
59+
# Handle PR branch URLs
60+
elif "/tree/" in repo_url:
61+
# Replace "github.com" with "raw.githubusercontent.com"
62+
raw_url = repo_url.replace("github.com", "raw.githubusercontent.com")
63+
# Replace "/tree/" with "/"
64+
raw_url = raw_url.replace("/tree/", "/")
65+
return raw_url
66+
67+
else:
68+
raise ValueError("Unsupported GitHub URL format.")
69+
except Exception as e:
70+
raise ValueError(f"Error converting GitHub URL to raw URL: {e}")
71+
72+
def download_files(base_url_or_path, download_dir):
73+
"""
74+
Download files from a raw GitHub base URL or copy files from a local directory.
75+
76+
Parameters:
77+
base_url_or_path (str): The base URL for raw files or the local directory path.
78+
download_dir (str): The directory to download/copy the files into.
79+
80+
Returns:
81+
str: The path to the downloaded/copied files.
82+
"""
83+
try:
84+
# Ensure the download directory exists
85+
os.makedirs(download_dir, exist_ok=True)
86+
87+
# List of files to download/copy (adjust as needed)
88+
files_to_download = [
89+
"schema.json",
90+
"examples/example.json",
91+
"examples/example-normalized.json",
92+
"examples/example.jsonld",
93+
"examples/example-normalized.jsonld",
94+
"ADOPTERS.yaml",
95+
"notes.yaml",
96+
]
97+
98+
if is_url(base_url_or_path):
99+
# Download files from a URL
100+
for file in files_to_download:
101+
file_url = f"{base_url_or_path.rstrip('/')}/{file}"
102+
file_path = os.path.join(download_dir, file)
103+
104+
# Ensure the directory structure exists
105+
os.makedirs(os.path.dirname(file_path), exist_ok=True)
106+
107+
# Download the file
108+
response = requests.get(file_url)
109+
if response.status_code == 200:
110+
with open(file_path, "wb") as f:
111+
f.write(response.content)
112+
else:
113+
raise Exception(f"Failed to download {file_url}: HTTP {response.status_code}")
114+
else:
115+
# Copy files from a local directory
116+
for file in files_to_download:
117+
src_path = os.path.join(base_url_or_path, file)
118+
dest_path = os.path.join(download_dir, file)
119+
120+
# Ensure the directory structure exists
121+
os.makedirs(os.path.dirname(dest_path), exist_ok=True)
122+
123+
# Copy the file
124+
if os.path.exists(src_path):
125+
shutil.copy(src_path, dest_path)
126+
else:
127+
raise Exception(f"File not found: {src_path}")
128+
129+
return download_dir
130+
except Exception as e:
131+
raise Exception(f"Error downloading/copying files: {e}")
132+
133+
def run_tests(test_files, repo_to_test, only_report_errors):
134+
"""
135+
Run a series of tests on a file.
136+
137+
Parameters:
138+
test_files (list): List of test module names (e.g., ["test_valid_json", "test_file_exists"]).
139+
repo_to_test (str): The path to the directory where the files are located.
140+
only_report_errors (bool): Whether to include only failed tests in the results.
141+
142+
Returns:
143+
dict: Results of the tests.
144+
"""
145+
results = {}
146+
for test_file in test_files:
147+
try:
148+
# Import the test module
149+
module = importlib.import_module(f"tests.{test_file}")
150+
# Run the test function (assumes the function name is the same as the module name without 'test_')
151+
test_function = getattr(module, test_file)
152+
test_name, success, message = test_function(repo_to_test)
153+
# Include the test result only if it failed or if only_report_errors is False
154+
if not only_report_errors or not success:
155+
results[test_file] = {
156+
"test_name": test_name,
157+
"success": success,
158+
"message": message
159+
}
160+
except Exception as e:
161+
results[test_file] = {
162+
"test_name": test_file,
163+
"success": False,
164+
"message": f"Error running test: {e}"
165+
}
166+
return results
167+
168+
if __name__ == "__main__":
169+
# Check if the repository URL/local path, email, and only_report_errors flag are provided as command-line arguments
170+
if len(sys.argv) != 4:
171+
print("Usage: python3 master_tests.py <repo_url_or_local_path> <email> <only_report_errors>")
172+
sys.exit(1)
173+
174+
# Get the repository URL/local path, email, and only_report_errors flag from the command-line arguments
175+
repo_url_or_local_path = sys.argv[1]
176+
email = sys.argv[2]
177+
only_report_errors = sys.argv[3].lower() == "true" or sys.argv[3] == "1"
178+
179+
# Validate the email (basic check)
180+
if not email or "@" not in email:
181+
print("Error: Missing or invalid email address.")
182+
sys.exit(1)
183+
184+
# Temporary directory to download/copy the files
185+
download_dir = "/var/html/www/tmp/repo_to_test"
186+
187+
try:
188+
# If the input is a URL, convert it to a raw file base URL
189+
if is_url(repo_url_or_local_path):
190+
raw_base_url = convert_github_url_to_raw(repo_url_or_local_path)
191+
else:
192+
raw_base_url = repo_url_or_local_path
193+
194+
# Download or copy the files
195+
repo_path = download_files(raw_base_url, download_dir)
196+
197+
# List of test files to run
198+
test_files = ["test_valid_json", "test_file_exists", "test_schema_descriptions", "test_schema_metadata", "test_duplicated_attributes"]
199+
200+
# Run the tests
201+
test_results = run_tests(test_files, repo_path, only_report_errors)
202+
203+
# Add email to the results
204+
test_results["email"] = email
205+
206+
# Display the results
207+
print(json.dumps(test_results, indent=4))
208+
209+
except Exception as e:
210+
print(f"Error: {e}")
211+
finally:
212+
# Clean up the temporary directory
213+
if os.path.exists(download_dir):
214+
shutil.rmtree(download_dir)

‎test_data_model/tests/__init__.py

Whitespace-only changes.

‎test_data_model/tests/test_duplicated_attributes.py

+103
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
#################################################################################
2+
# Licensed to the FIWARE Foundation (FF) under one #
3+
# or more contributor license agreements. The FF licenses this file #
4+
# to you under the Apache License, Version 2.0 (the "License") #
5+
# you may not use this file except in compliance with the License. #
6+
# You may obtain a copy of the License at #
7+
# #
8+
# http://www.apache.org/licenses/LICENSE-2.0 #
9+
# #
10+
# Unless required by applicable law or agreed to in writing, software #
11+
# distributed under the License is distributed on an "AS IS" BASIS, #
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
13+
# See the License for the specific language governing permissions and #
14+
# limitations under the License. #
15+
# Author: Alberto Abella #
16+
#################################################################################
17+
import json
18+
import os
19+
20+
def find_duplicates_in_object(obj, path="", duplicates=None):
21+
"""
22+
Recursively find duplicated attributes in a JSON object.
23+
24+
Parameters:
25+
obj (dict): The JSON object to check.
26+
path (str): The current path in the JSON structure (used for reporting).
27+
duplicates (list): A list to store duplicate attribute paths.
28+
29+
Returns:
30+
list: A list of duplicate attribute paths.
31+
"""
32+
if duplicates is None:
33+
duplicates = []
34+
35+
# Track attribute names at the current level
36+
attribute_counts = {}
37+
38+
for key, value in obj.items():
39+
# Update the count for the current attribute
40+
if key in attribute_counts:
41+
attribute_counts[key] += 1
42+
else:
43+
attribute_counts[key] = 1
44+
45+
# If the attribute is duplicated at this level, add it to the duplicates list
46+
if attribute_counts[key] > 1:
47+
duplicates.append(f"{path}.{key}" if path else key)
48+
49+
# If the value is a dictionary, recursively check for duplicates
50+
if isinstance(value, dict):
51+
find_duplicates_in_object(value, f"{path}.{key}" if path else key, duplicates)
52+
53+
return duplicates
54+
55+
def test_duplicated_attributes(repo_path):
56+
"""
57+
Test if any attributes in the schema.json file are duplicated at the same object level.
58+
59+
Parameters:
60+
repo_path (str): The path to the repository containing the schema.json file.
61+
62+
Returns:
63+
tuple: (test_name: str, success: bool, output: list)
64+
"""
65+
test_name = "Checking for duplicated attributes in schema.json"
66+
success = True
67+
output = []
68+
69+
try:
70+
# Load the schema.json file
71+
with open(f"{repo_path}/schema.json", 'r') as file:
72+
schema = json.load(file)
73+
74+
# Find duplicates in the schema
75+
duplicates = find_duplicates_in_object(schema)
76+
77+
# Report duplicates
78+
if duplicates:
79+
success = False
80+
output.append("*** Duplicated attributes found:")
81+
for duplicate in duplicates:
82+
output.append(f"*** - {duplicate}")
83+
else:
84+
output.append("No duplicated attributes found.")
85+
86+
except json.JSONDecodeError:
87+
success = False
88+
output.append("*** schema.json is not a valid JSON file")
89+
except FileNotFoundError:
90+
success = False
91+
output.append("*** schema.json file not found")
92+
93+
return test_name, success, output
94+
#
95+
# if __name__ == "__main__":
96+
# # Example usage
97+
# repo_path = "/path/to/repo"
98+
# test_name, success, output = test_duplicated_attributes(repo_path)
99+
# print(f"Test: {test_name}")
100+
# print(f"Success: {success}")
101+
# print("Output:")
102+
# for line in output:
103+
# print(line)

‎test_data_model/tests/test_file_exists.py

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#################################################################################
2+
# Licensed to the FIWARE Foundation (FF) under one #
3+
# or more contributor license agreements. The FF licenses this file #
4+
# to you under the Apache License, Version 2.0 (the "License") #
5+
# you may not use this file except in compliance with the License. #
6+
# You may obtain a copy of the License at #
7+
# #
8+
# http://www.apache.org/licenses/LICENSE-2.0 #
9+
# #
10+
# Unless required by applicable law or agreed to in writing, software #
11+
# distributed under the License is distributed on an "AS IS" BASIS, #
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
13+
# See the License for the specific language governing permissions and #
14+
# limitations under the License. #
15+
# Author: Alberto Abella #
16+
#################################################################################
17+
import os
18+
19+
def test_file_exists(repo_path):
20+
"""
21+
Test if a file exists.
22+
23+
Parameters:
24+
file_path (str): The path to the file to check.
25+
26+
Returns:
27+
tuple: (success: bool, message: str)
28+
"""
29+
30+
test_name = "Checking if the mandatory files according to the contribution manual are present"
31+
32+
mandatory_files = ["schema.json", "examples/example.json", "examples/example-normalized.json", "examples/example.jsonld", "examples/example-normalized.jsonld", "notes.yaml", "ADOPTERS.yaml"]
33+
output = []
34+
success = True
35+
for file in mandatory_files:
36+
path_to_file = repo_path + "/" + file
37+
# print(f"The path to file is {path_to_file}")
38+
exist_file = os.path.exists(path_to_file)
39+
success = success and exist_file
40+
41+
if exist_file:
42+
output.append(f"The file {path_to_file} exists")
43+
else:
44+
output.append(f"*** The file {path_to_file} DOES NOT exist")
45+
return test_name, success, output
46+

0 commit comments

Comments
 (0)