Skip to content

Commit a6c9c95

Browse files
himynamesdavehimynamesdave
committed
adding some import scripts
1 parent 72b7790 commit a6c9c95

7 files changed

+525
-4
lines changed

‎README.md

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -80,14 +80,14 @@ You can access the django admin UI at:
8080

8181
http://127.0.0.1:8005/admin
8282

83-
### Add data
84-
85-
By default, the `ARANGODB_DATABASE``
86-
8783
### Running in production
8884

8985
Note, if you intend on using this in production, you should also modify the variables in the `.env` file for `POSTGRES_DB`, `POSTGRES_USER`, `POSTGRES_PASS`, `DJANGO_SECRET` and `DEBUG` (to `False`)
9086

87+
## Quickstart
88+
89+
To get up and running quickly head to `/utilities/README.md` for some scripts that will automate the backfill of data.
90+
9191
## Support
9292

9393
[Minimal support provided via the DOGESEC community](https://community.dogesec.com/).

‎utilities/README.md

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
# Vulmatch Utilities
2+
3+
Run these to backfill Vulmatch with data.
4+
5+
6+
## Enrichment backfill
7+
8+
### CPE
9+
10+
Generally very old CPEs are no longer observed, but that does not mean that recent CVEs will reference them.
11+
12+
To be safe backfill all CPEs (beware, this is over 1 million records).
13+
14+
If you want, you can also specify an earliest CPE date in the script below. Be aware though, if CVEs reference CPEs you havent imported (because they have a modified time earlier than that specified) you will miss the CVE -> CPE joins.
15+
16+
### ATT&CK Enterprise
17+
18+
Import all available versions (recommended)
19+
20+
```shell
21+
python3 import_attack_enterprise_archive.py
22+
```
23+
24+
Import specific versions
25+
26+
```shell
27+
python3 import_attack_enterprise_archive.py 1.0 14.1 15.0 15.1
28+
```
29+
30+
### ATT&CK ICS
31+
32+
Import all available versions (recommended)
33+
34+
```shell
35+
python3 import_attack_ics_archive.py
36+
```
37+
38+
Import specific versions
39+
40+
```shell
41+
python3 import_attack_ics_archive.py 14.1 15.0 15.1
42+
```
43+
44+
### ATT&CK Mobile
45+
46+
Import all available versions (recommended)
47+
48+
```shell
49+
python3 import_attack_mobile_archive.py
50+
```
51+
52+
Import specific versions
53+
54+
```shell
55+
python3 import_attack_mobile_archive.py 15.1 11.1-beta
56+
```
57+
58+
### CWE
59+
60+
Import all available versions (recommended)
61+
62+
```shell
63+
python3 import_cwe_archive.py
64+
```
65+
66+
Import specific versions
67+
68+
```shell
69+
python3 import_cwe_archive.py 4.14 4.15
70+
```
71+
72+
### CAPEC
73+
74+
Import all available versions (recommended)
75+
76+
```shell
77+
python3 import_capec_archive.py
78+
```
79+
80+
Import specific versions
81+
82+
```shell
83+
python3 import_capec_archive.py 3.8 3.9
84+
```

‎utilities/import_attack_enterprise_archive.py

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
import argparse
2+
import requests
3+
import time
4+
5+
# Base URLs of the API
6+
BASE_URL = 'http://127.0.0.1:8005/api/v1/attack-enterprise/'
7+
JOB_STATUS_URL = 'http://127.0.0.1:8005/api/v1/jobs/'
8+
9+
# List of all available versions
10+
ALL_VERSIONS = [
11+
"1.0", "2.0", "3.0", "4.0", "5.0", "5.1", "5.2", "6.0", "6.1", "6.2", "6.3",
12+
"7.0", "7.1", "7.2", "8.0", "8.1", "8.2", "9.0", "10.0", "10.1", "11.0",
13+
"11.1", "11.2", "11.3", "12.0", "12.1", "13.0", "13.1", "14.0", "14.1",
14+
"15.0", "15.1"
15+
]
16+
17+
# Function to post version and get job ID
18+
def post_version(version):
19+
url = BASE_URL
20+
headers = {
21+
'accept': 'application/json',
22+
'Content-Type': 'application/json'
23+
}
24+
25+
# Replace . with _ for version formatting
26+
version_str = str(version).replace('.', '_')
27+
data = {
28+
"version": version_str
29+
}
30+
31+
print(f"Sending POST request for version: {version_str}")
32+
response = requests.post(url, headers=headers, json=data)
33+
34+
# Print full request and response for debugging
35+
print(f"Request Data: {data}")
36+
print(f"Response: {response.status_code}, {response.text}")
37+
38+
# Accept both 200 OK and 201 Created as successful responses
39+
if response.status_code in [200, 201]:
40+
response_data = response.json()
41+
return response_data['id'] # Return job ID
42+
else:
43+
raise Exception(f"Failed to submit version {version}: {response.status_code} - {response.text}")
44+
45+
# Function to check job status
46+
def check_job_status(job_id):
47+
url = f"{JOB_STATUS_URL}{job_id}/"
48+
49+
while True:
50+
print(f"Checking job status for job ID: {job_id}")
51+
response = requests.get(url)
52+
53+
# Print full request and response for debugging
54+
print(f"Job Status Response: {response.status_code}, {response.text}")
55+
56+
if response.status_code == 200:
57+
response_data = response.json()
58+
if response_data['state'] == 'completed':
59+
print(f"Job {job_id} completed.")
60+
return
61+
else:
62+
print(f"Job {job_id} still in progress. Waiting...")
63+
time.sleep(30) # Wait 30 seconds before checking again
64+
else:
65+
raise Exception(f"Failed to check job status: {response.status_code} - {response.text}")
66+
67+
def main():
68+
# Parse CLI arguments
69+
parser = argparse.ArgumentParser(description="Post versions and track job status.")
70+
parser.add_argument('versions', nargs='*', type=float, help="List of versions to post as numbers (e.g., 14.1, 15.0). If not provided, all versions will be imported.")
71+
args = parser.parse_args()
72+
73+
# Use provided versions or default to all if none are provided
74+
versions = sorted(args.versions) if args.versions else sorted(ALL_VERSIONS)
75+
76+
# Post each version and check job status
77+
for version in versions:
78+
try:
79+
job_id = post_version(version)
80+
check_job_status(job_id)
81+
except Exception as e:
82+
print(f"Error occurred: {e}")
83+
break
84+
85+
if __name__ == "__main__":
86+
main()

‎utilities/import_attack_ics_archive.py

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
import argparse
2+
import requests
3+
import time
4+
5+
# Base URLs of the API
6+
BASE_URL = 'http://127.0.0.1:8005/api/v1/attack-ics/'
7+
JOB_STATUS_URL = 'http://127.0.0.1:8005/api/v1/jobs/'
8+
9+
# List of all available versions
10+
ALL_VERSIONS = [
11+
"8.0", "8.1", "8.2", "9.0", "10.0", "10.1", "11.0",
12+
"11.1", "11.2", "11.3", "12.0", "12.1", "13.0", "13.1", "14.0", "14.1",
13+
"15.0", "15.1"
14+
]
15+
16+
# Function to post version and get job ID
17+
def post_version(version):
18+
url = BASE_URL
19+
headers = {
20+
'accept': 'application/json',
21+
'Content-Type': 'application/json'
22+
}
23+
24+
# Replace . with _ for version formatting
25+
version_str = str(version).replace('.', '_')
26+
data = {
27+
"version": version_str
28+
}
29+
30+
print(f"Sending POST request for version: {version_str}")
31+
response = requests.post(url, headers=headers, json=data)
32+
33+
# Print full request and response for debugging
34+
print(f"Request Data: {data}")
35+
print(f"Response: {response.status_code}, {response.text}")
36+
37+
# Accept both 200 OK and 201 Created as successful responses
38+
if response.status_code in [200, 201]:
39+
response_data = response.json()
40+
return response_data['id'] # Return job ID
41+
else:
42+
raise Exception(f"Failed to submit version {version}: {response.status_code} - {response.text}")
43+
44+
# Function to check job status
45+
def check_job_status(job_id):
46+
url = f"{JOB_STATUS_URL}{job_id}/"
47+
48+
while True:
49+
print(f"Checking job status for job ID: {job_id}")
50+
response = requests.get(url)
51+
52+
# Print full request and response for debugging
53+
print(f"Job Status Response: {response.status_code}, {response.text}")
54+
55+
if response.status_code == 200:
56+
response_data = response.json()
57+
if response_data['state'] == 'completed':
58+
print(f"Job {job_id} completed.")
59+
return
60+
else:
61+
print(f"Job {job_id} still in progress. Waiting...")
62+
time.sleep(30) # Wait 30 seconds before checking again
63+
else:
64+
raise Exception(f"Failed to check job status: {response.status_code} - {response.text}")
65+
66+
def main():
67+
# Parse CLI arguments
68+
parser = argparse.ArgumentParser(description="Post versions and track job status.")
69+
parser.add_argument('versions', nargs='*', type=float, help="List of versions to post as numbers (e.g., 14.1, 15.0). If not provided, all versions will be imported.")
70+
args = parser.parse_args()
71+
72+
# Use provided versions or default to all if none are provided
73+
versions = sorted(args.versions) if args.versions else sorted(ALL_VERSIONS)
74+
75+
# Post each version and check job status
76+
for version in versions:
77+
try:
78+
job_id = post_version(version)
79+
check_job_status(job_id)
80+
except Exception as e:
81+
print(f"Error occurred: {e}")
82+
break
83+
84+
if __name__ == "__main__":
85+
main()

‎utilities/import_attack_mobile_archive.py

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
import argparse
2+
import requests
3+
import time
4+
5+
# Base URLs of the API
6+
BASE_URL = 'http://127.0.0.1:8005/api/v1/attack-mobile/'
7+
JOB_STATUS_URL = 'http://127.0.0.1:8005/api/v1/jobs/'
8+
9+
# List of all available versions
10+
ALL_VERSIONS = [
11+
"1.0", "2.0", "3.0", "4.0", "5.0", "5.1", "5.2", "6.0", "6.1", "6.2", "6.3",
12+
"7.0", "7.1", "7.2", "8.0", "8.1", "8.2", "9.0", "10.0", "10.1", "11.0-beta",
13+
"11.1-beta", "11.2-beta", "11.3", "12.0", "12.1", "13.0", "13.1", "14.0", "14.1",
14+
"15.0", "15.1"
15+
]
16+
17+
# Function to post version and get job ID
18+
def post_version(version):
19+
url = BASE_URL
20+
headers = {
21+
'accept': 'application/json',
22+
'Content-Type': 'application/json'
23+
}
24+
25+
# Replace . with _ for version formatting
26+
version_str = str(version).replace('.', '_')
27+
data = {
28+
"version": version_str
29+
}
30+
31+
print(f"Sending POST request for version: {version_str}")
32+
response = requests.post(url, headers=headers, json=data)
33+
34+
# Print full request and response for debugging
35+
print(f"Request Data: {data}")
36+
print(f"Response: {response.status_code}, {response.text}")
37+
38+
# Accept both 200 OK and 201 Created as successful responses
39+
if response.status_code in [200, 201]:
40+
response_data = response.json()
41+
return response_data['id'] # Return job ID
42+
else:
43+
raise Exception(f"Failed to submit version {version}: {response.status_code} - {response.text}")
44+
45+
# Function to check job status
46+
def check_job_status(job_id):
47+
url = f"{JOB_STATUS_URL}{job_id}/"
48+
49+
while True:
50+
print(f"Checking job status for job ID: {job_id}")
51+
response = requests.get(url)
52+
53+
# Print full request and response for debugging
54+
print(f"Job Status Response: {response.status_code}, {response.text}")
55+
56+
if response.status_code == 200:
57+
response_data = response.json()
58+
if response_data['state'] == 'completed':
59+
print(f"Job {job_id} completed.")
60+
return
61+
else:
62+
print(f"Job {job_id} still in progress. Waiting...")
63+
time.sleep(30) # Wait 30 seconds before checking again
64+
else:
65+
raise Exception(f"Failed to check job status: {response.status_code} - {response.text}")
66+
67+
def main():
68+
# Parse CLI arguments
69+
parser = argparse.ArgumentParser(description="Post versions and track job status.")
70+
parser.add_argument('versions', nargs='*', type=str, help="List of versions to post (e.g., 14.1, 15.0, 11.1-beta). If not provided, all versions will be imported.")
71+
args = parser.parse_args()
72+
73+
# Use provided versions or default to all if none are provided
74+
versions = sorted(args.versions) if args.versions else sorted(ALL_VERSIONS, key=lambda v: [int(x) if x.isdigit() else x for x in v.replace('-', '.').split('.')])
75+
76+
# Post each version and check job status
77+
for version in versions:
78+
try:
79+
job_id = post_version(version)
80+
check_job_status(job_id)
81+
except Exception as e:
82+
print(f"Error occurred: {e}")
83+
break
84+
85+
if __name__ == "__main__":
86+
main()

0 commit comments

Comments
 (0)