diff --git a/.gitignore b/.gitignore index 27b9660c..4f8ac831 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,6 @@ nb-configuration.xml **/nbproject/ local.properties data-source/data/ + +# IntellIJ run configs +.run diff --git a/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/nvd/NvdCveClientBuilder.java b/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/nvd/NvdCveClientBuilder.java index 51b98e77..9447d3a1 100644 --- a/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/nvd/NvdCveClientBuilder.java +++ b/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/nvd/NvdCveClientBuilder.java @@ -17,6 +17,7 @@ package io.github.jeremylong.openvulnerability.client.nvd; import io.github.jeremylong.openvulnerability.client.HttpAsyncClientSupplier; +import java.util.stream.Collectors; import org.apache.hc.core5.http.NameValuePair; import org.apache.hc.core5.http.message.BasicNameValuePair; import org.slf4j.Logger; @@ -224,12 +225,22 @@ public NvdCveClientBuilder withFilter(BooleanFilter filter) { * @return the builder */ public NvdCveClientBuilder withLastModifiedFilter(ZonedDateTime utcStartDate, ZonedDateTime utcEndDate) { - DateTimeFormatter dtf = DateTimeFormatter.ofPattern("uuuu-MM-dd'T'HH:mm:ssX"); + DateTimeFormatter dtf = DateTimeFormatter.ISO_OFFSET_DATE_TIME; + // ensure we have no filters yet + removeLastModifiedFilter(); + filters.add(new BasicNameValuePair("lastModStartDate", utcStartDate.format(dtf))); filters.add(new BasicNameValuePair("lastModEndDate", utcEndDate.format(dtf))); return this; } + public NvdCveClientBuilder removeLastModifiedFilter() { + // ensure we have no filters yet + filters.removeIf((item) -> item.getName().equals("lastModStartDate")); + filters.removeIf((item) -> item.getName().equals("lastModEndDate")); + return this; + } + /** * Use an additional identifier as part of the User-Agent when making requests. * @@ -249,12 +260,22 @@ public NvdCveClientBuilder withAdditionalUserAgent(String userAgent) { * @return the builder */ public NvdCveClientBuilder withPublishedDateFilter(ZonedDateTime utcStartDate, ZonedDateTime utcEndDate) { - DateTimeFormatter dtf = DateTimeFormatter.ofPattern("uuuu-MM-dd'T'HH:mm:ssX"); + DateTimeFormatter dtf = DateTimeFormatter.ISO_OFFSET_DATE_TIME; + + // ensure we have no filters yet + removePublishDateFilter(); + filters.add(new BasicNameValuePair("pubStartDate", utcStartDate.format(dtf))); filters.add(new BasicNameValuePair("pubEndDate", utcEndDate.format(dtf))); return this; } + public NvdCveClientBuilder removePublishDateFilter() { + filters.removeIf((item) -> item.getName().equals("pubStartDate")); + filters.removeIf((item) -> item.getName().equals("pubEndDate")); + return this; + } + /** * Filter the results for a specific CVSS V2 Severity. * diff --git a/vulnz/Dockerfile b/vulnz/Dockerfile index 181ee72d..8f108a14 100644 --- a/vulnz/Dockerfile +++ b/vulnz/Dockerfile @@ -39,12 +39,11 @@ COPY ["/src/docker/supervisor/supervisord.conf", "/etc/supervisord.conf"] COPY ["/src/docker/scripts/mirror.sh", "/mirror.sh"] COPY ["/src/docker/scripts/validate.sh", "/validate.sh"] COPY ["/src/docker/crontab/mirror", "/etc/crontabs/mirror"] -COPY ["/src/docker/crontab/validate", "/etc/crontabs/validate"] COPY ["/src/docker/apache/mirror.conf", "/usr/local/apache2/conf"] COPY ["/build/libs/vulnz-$BUILD_VERSION.jar", "/usr/local/bin/vulnz"] RUN chmod +x /mirror.sh /validate.sh && \ - chown root:root /etc/crontabs/mirror /etc/crontabs/validate && \ + chown root:root /etc/crontabs/mirror && \ chown mirror:mirror /mirror.sh /validate.sh && \ chown mirror:mirror /usr/local/bin/vulnz diff --git a/vulnz/README.md b/vulnz/README.md index 3b9278e0..253a3f4b 100644 --- a/vulnz/README.md +++ b/vulnz/README.md @@ -132,7 +132,6 @@ docker run --name vulnz -e JAVA_OPT=-Xmx2g jeremylong/open-vulnerability-data-mi # you can also adjust the delay docker run --name vulnz -e NVD_API_KEY=myapikey -e DELAY=3000 jeremylong/open-vulnerability-data-mirror:v7.2.1 - ``` If you like, run this to pre-populate the database right away @@ -148,7 +147,8 @@ Assuming the current version is `7.2.1` ```bash export TARGET_VERSION=7.2.1 ./gradlew vulnz:build -Pversion=$TARGET_VERSION -docker build vulnz/ -t ghcr.io/jeremylong/vulnz:$TARGET_VERSION --build-arg BUILD_VERSION=$TARGET_VERSION +docker build vulnz/ -t ghcr.io/jeremylong/vulnz:v$TARGET_VERSION --build-arg BUILD_VERSION=$TARGET_VERSION +docker push ``` ### Release diff --git a/vulnz/src/docker/crontab/mirror b/vulnz/src/docker/crontab/mirror index e0a95541..93111d7a 100755 --- a/vulnz/src/docker/crontab/mirror +++ b/vulnz/src/docker/crontab/mirror @@ -1 +1,2 @@ 0 0 * * * /mirror.sh 2>&1 | tee -a /var/log/docker_out.log | tee -a /var/log/cron_mirror.log +0 4 * * * /validate.sh 2>&1 | tee -a /var/log/docker_out.log | tee -a /var/log/cron_validate.log diff --git a/vulnz/src/docker/crontab/validate b/vulnz/src/docker/crontab/validate deleted file mode 100644 index 6dbbb593..00000000 --- a/vulnz/src/docker/crontab/validate +++ /dev/null @@ -1 +0,0 @@ -0 4 * * * /validate.sh 2>&1 | tee -a /var/log/docker_out.log | tee -a /var/log/cron_validate.log diff --git a/vulnz/src/docker/scripts/mirror.sh b/vulnz/src/docker/scripts/mirror.sh index 574bf0ec..51ee4063 100755 --- a/vulnz/src/docker/scripts/mirror.sh +++ b/vulnz/src/docker/scripts/mirror.sh @@ -1,8 +1,17 @@ #!/bin/sh + set -e echo "Updating..." +LOCKFILE=/tmp/vulzn.lock + +if [ -f $LOCKFILE ]; then + echo "Lockfile found - another mirror-sync process already running" +else + touch $LOCKFILE +fi + DELAY_ARG="" if [ -z $NVD_API_KEY ]; then DELAY_ARG="--delay=10000" @@ -33,5 +42,12 @@ if [ -n "${DEBUG}" ]; then DEBUG_ARG="--debug" fi -exec java $JAVA_OPT -jar /usr/local/bin/vulnz cve $DELAY_ARG $DEBUG_ARG $MAX_RETRY_ARG $MAX_RECORDS_PER_PAGE_ARG --cache --directory /usr/local/apache2/htdocs +function remove_lockfile() { + rm -f $LOCKFILE + exit 0 +} +trap remove_lockfile SIGHUP SIGINT SIGQUIT SIGABRT SIGALRM SIGTERM SIGTSTP + +java $JAVA_OPT -jar /usr/local/bin/vulnz cve $DELAY_ARG $DEBUG_ARG $MAX_RETRY_ARG $MAX_RECORDS_PER_PAGE_ARG $CONTINUE_ARG --cache --directory /usr/local/apache2/htdocs +rm -f $LOCKFILE diff --git a/vulnz/src/docker/supervisor/supervisord.conf b/vulnz/src/docker/supervisor/supervisord.conf index 00fc65d6..e7cdedfd 100644 --- a/vulnz/src/docker/supervisor/supervisord.conf +++ b/vulnz/src/docker/supervisor/supervisord.conf @@ -59,3 +59,4 @@ stdout_logfile=/dev/fd/1 stdout_logfile_maxbytes=0 redirect_stderr=true user=mirror +stopsecs=29 diff --git a/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java b/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java index 8d3acd9d..650025fa 100644 --- a/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java +++ b/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java @@ -32,39 +32,29 @@ import io.github.jeremylong.vulnz.cli.cache.CacheException; import io.github.jeremylong.vulnz.cli.cache.CacheProperties; import io.github.jeremylong.vulnz.cli.model.BasicOutput; +import io.github.jeremylong.vulnz.cli.model.CvesNvdPojo; import io.github.jeremylong.vulnz.cli.ui.IProgressMonitor; import io.github.jeremylong.vulnz.cli.ui.JlineShutdownHook; import io.github.jeremylong.vulnz.cli.ui.ProgressMonitor; import io.prometheus.metrics.core.metrics.Gauge; -import it.unimi.dsi.fastutil.objects.Object2ObjectMap; -import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap; +import java.io.*; +import java.nio.file.Path; +import java.time.*; +import java.time.temporal.ChronoUnit; +import java.util.*; +import java.util.stream.Collectors; +import java.util.zip.GZIPInputStream; import org.apache.commons.io.output.CountingOutputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import picocli.CommandLine; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.io.PrintWriter; import java.nio.charset.StandardCharsets; import java.security.DigestOutputStream; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; -import java.time.Year; -import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; -import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; import static com.diogonunes.jcolor.Ansi.colorize; @@ -76,11 +66,41 @@ public class CveCommand extends AbstractNvdCommand { * Reference to the logger. */ private static final Logger LOG = LoggerFactory.getLogger(CveCommand.class); + + /** + * Start year (until today) to cache CVEs for. + */ + private static final int START_YEAR = 2002; + + // FIXME - get format and version from API + private static final String FORMAT = "NVD_CVE"; + private static final String VERSION = "2.0"; + + /** + * by NVDs API limit, 120 is the max range + */ + private static final int NVD_API_MAX_DAYS_RANGE = 115; + + private static final int EXIT_CODE_SUCCESS = 0; + /** + * At least one year failed to fetch + */ + private static final int EXIT_CODE_PARTIAL_ERROR_YEAR = 100; + + /** + * At least one year failed to fetch + */ + private static final int EXIT_CODE_FATAL_ERROR_RECENT_MODIFIED = 101; /** * Hex code characters used in getHex. */ private static final String HEXES = "0123456789abcdef"; + /** + * Defines how many days we look into the past when aggregating the recent-modified cache + */ + private static final long MAX_AGE_OF_MODIFIED_DIFF = 8; + private static final Gauge CVE_LOAD_COUNTER = Gauge.builder().name("cve_load_counter") .help("Total number of loaded cve's").register(); private static final Gauge CVE_COUNTER = Gauge.builder().name("cve_counter").help("Total number of cached cve's") @@ -229,16 +249,6 @@ public Integer timedCall() throws Exception { if (configGroup != null && configGroup.cacheSettings != null) { CacheProperties properties = new CacheProperties(configGroup.cacheSettings.directory); - if (properties.has("lastModifiedDate")) { - ZonedDateTime start = properties.getTimestamp("lastModifiedDate"); - ZonedDateTime end = start.minusDays(-120); - if (end.compareTo(ZonedDateTime.now()) > 0) { - builder.withLastModifiedFilter(start, end); - } else { - LOG.warn( - "Requesting the entire set of NVD CVE data via the api as the cache was last updated over 120 days ago"); - } - } if (configGroup.cacheSettings.prefix != null) { properties.set("prefix", configGroup.cacheSettings.prefix); } @@ -262,130 +272,296 @@ public Integer timedCall() throws Exception { return processRequest(builder); } - private Integer processRequest(NvdCveClientBuilder builder, CacheProperties properties) { + /** + * For all years, fetch the NVD vulnerabilities and save them each into one cache file. + */ + private Integer processRequest(NvdCveClientBuilder apiBuilder, CacheProperties properties) { + int exitCode = EXIT_CODE_SUCCESS; + // will hold all entries that have been changed within the last8 days across all years + List recentlyChangedEntries = new ArrayList<>(); + + for (int currentYear = START_YEAR; currentYear <= Year.now().getValue(); currentYear++) { + + // Be forgiving, if we fail to fetch a year, we just continue with the next one + try { + // reset our filters for each year + apiBuilder.removeLastModifiedFilter(); + apiBuilder.removePublishDateFilter(); + + Path cacheFilePath = buildCacheTargetFileForYear(properties, currentYear); + LOG.info("INFO *** Processing year {} ***", currentYear); + CvesNvdPojo existingCacheData = loadExistingCacheAndConfigureApi(apiBuilder, currentYear, + cacheFilePath); + CvesNvdPojo cvesForYear = aggregateCvesForYear(currentYear, apiBuilder); + + // merge old with new data. It is intended to add old items to the new ones, thus we keep newly fetched + // items with the same ID from the API, not overriding from old cache + if (existingCacheData != null) { + LOG.info("INFO Fetched #{} updated entries for already existing local cache with #{} entries", + cvesForYear.vulnerabilities.size(), existingCacheData.vulnerabilities.size()); + cvesForYear.vulnerabilities.addAll(existingCacheData.vulnerabilities); + } else { + LOG.info("INFO Fetched #{} new entries for year {}", cvesForYear.vulnerabilities.size(), + currentYear); + } + + if (cvesForYear.lastUpdated != null) { + properties.set("lastModifiedDate", cvesForYear.lastUpdated); + } + storeEntireYearToCache(currentYear, cvesForYear, properties); + LOG.info("INFO *** Finished year {} with #{} entries ***", currentYear, + cvesForYear.vulnerabilities.size()); + + // remove all entries older than MAX_AGE_OF_MODIFIED_DIFF days + removeOutdatedCves(cvesForYear.vulnerabilities); + recentlyChangedEntries.addAll(cvesForYear.vulnerabilities); + } catch (Exception ex) { + LOG.error("\nPARTIAL-ERROR processing year {}", currentYear, ex); + LOG.info("INFO ... continuing with next year"); + exitCode = EXIT_CODE_PARTIAL_ERROR_YEAR; + } + } + + try { + saveRecentlyModifiedCacheFile(recentlyChangedEntries, properties, ZonedDateTime.now()); + } catch (Exception ex) { + LOG.error("\nERROR saving recent-modified file", ex); + exitCode = EXIT_CODE_FATAL_ERROR_RECENT_MODIFIED; + } + + return exitCode; + } + + private CvesNvdPojo loadExistingCacheAndConfigureApi(NvdCveClientBuilder apiBuilder, int currentYear, + Path cacheFilePath) { + ZonedDateTime lastUpdateDate = determineExistingCacheFileLastChanged(cacheFilePath); + if (lastUpdateDate == null) { + // no existing cache exists - nothing to load and nothing to configure. Fetch the entire year. + return null; + } + + // else only fetch entries that have been changed since the last update + apiBuilder.withLastModifiedFilter(lastUpdateDate, ZonedDateTime.now()); + LOG.info("INFO Found existing local cache for year {}. Cache was created/updated on {}", currentYear, + lastUpdateDate.format(DateTimeFormatter.RFC_1123_DATE_TIME)); + LOG.info("INFO Only fetching items that have been changed since then from the API"); + + // Load existing cache data + return loadCvesFromCache(cacheFilePath); + } + + /** + * Given the year, fetch all vulnerabilities for this year by using a publication date range filter with a maximum + * range of 120 days (due to NVD api limits). Ensures vulnerabilities are unique and sorted by publishing date, ASC + */ + private CvesNvdPojo aggregateCvesForYear(int year, NvdCveClientBuilder apiBuilder) { + + int sliceStartDay = 1; + + ZonedDateTime yearFirstDay = ZonedDateTime.of(year, 1, 1, 0, 0, 0, 0, ZoneId.systemDefault()); + int yearLengthInDays = Year.from(yearFirstDay).length(); + CvesNvdPojo finalResult = new CvesNvdPojo(new ArrayList<>(), ZonedDateTime.now()); + while (sliceStartDay < yearLengthInDays) { + // first day of the year we start from + ZonedDateTime from = yearFirstDay.plusDays(sliceStartDay); + + if (from.isAfter(ZonedDateTime.now())) { + // we are fetching for items in the future, stop here + break; + } + + // get the end day of the range, but ensure we do not overshoot + int sliceEndDay = Math.min(sliceStartDay + NVD_API_MAX_DAYS_RANGE, yearLengthInDays - 1); + ZonedDateTime to = yearFirstDay.plusDays(sliceEndDay) + // since we need the end of the day, just pick the next one and reset time to 00:00 + .plusDays(1).truncatedTo(ChronoUnit.DAYS); + + // apply our "year slice" range to the API call + apiBuilder.withPublishedDateFilter(from, to); + + // fetch entries for this range + CvesNvdPojo currentResult = fetchFromNVDApi(apiBuilder); + + // update our last updated, since we crawl up the year, this will be the most recent in the end + finalResult.lastUpdated = currentResult.lastUpdated; + // aggregate all results + finalResult.vulnerabilities.addAll(currentResult.vulnerabilities); + + // let the next fetch start on the following day we ended on before (since we fetched up to 'to midnight') + sliceStartDay = sliceEndDay + 1; + } + + return finalResult; + } + + /** + * Given the CVEs for the year, stores all of them in a json cache file and a meta-dat file. creates + * nvdcve-$year.json.gz and nvdcve-$year.meta + */ + private void storeEntireYearToCache(int year, CvesNvdPojo cves, CacheProperties properties) { + ZonedDateTime lastChanged = Objects.requireNonNullElseGet(cves.lastUpdated, ZonedDateTime::now); + properties.set("lastModifiedDate." + year, lastChanged); + int size = cves.vulnerabilities.size(); + MessageDigest md = getDigestAlg(); + + // save vulnerabilities into cache + CveApiJson20 data = new CveApiJson20(size, 0, size, FORMAT, VERSION, lastChanged, cves.vulnerabilities); + final File cacheFile = buildCacheTargetFileForYear(properties, year).toFile(); + long uncompressedSize = saveCvesToCache(cacheFile, data, md); + + // save meta data + final File metaDataFile = buildMetaDataTargetFileForYear(properties, year).toFile(); + saveMetaData(metaDataFile, cacheFile.length(), uncompressedSize, lastChanged, md); + } + + /** + * Create cache holding all CVE items that have been changed within the last 8 days. Creates: + * nvdcve-modified.json.gz and nvdcve-modified.meta + */ + private void saveRecentlyModifiedCacheFile(List recentlyChanged, CacheProperties properties, + ZonedDateTime lastChanged) { + final String prefix = properties.get("prefix", "nvdcve-"); + Path recentChangesCachePath = Path.of(properties.getDirectory().getPath(), prefix + "modified.json.gz"); + + // preload the old modified + var oldModifiedCacheData = loadCvesFromCache(recentChangesCachePath); + + // merge newly entries with the previously stored ones + oldModifiedCacheData.vulnerabilities.addAll(recentlyChanged); + // ensure we do not include a slice more the + removeOutdatedCves(oldModifiedCacheData.vulnerabilities); + + int recentSize = oldModifiedCacheData.vulnerabilities.size(); + CveApiJson20 data = new CveApiJson20(recentSize, 0, recentSize, FORMAT, VERSION, lastChanged, + oldModifiedCacheData.vulnerabilities); + + MessageDigest md = getDigestAlg(); + + // create cache file including the CVE entries that recently changed + File recentCacheFile = recentChangesCachePath.toFile(); + long uncompressedSize = saveCvesToCache(recentCacheFile, data, md); + LOG.info("INFO Stored {} entries in {} as recent changed items across all years", + data.getVulnerabilities().size(), recentCacheFile.getName()); + + // Create meta-file + Path metaDataFile = Path.of(properties.getDirectory().getPath(), prefix + "modified.meta"); + saveMetaData(metaDataFile.toFile(), recentCacheFile.length(), uncompressedSize, lastChanged, md); + } + + private CvesNvdPojo loadCvesFromCache(Path cacheFilePath) { + // Load existing cache data + ObjectMapper objectMapper = getCacheObjectMapper(); + try (FileInputStream fis = new FileInputStream(cacheFilePath.toFile()); + GZIPInputStream gzis = new GZIPInputStream(fis)) { + CveApiJson20 data = objectMapper.readValue(gzis, CveApiJson20.class); + return new CvesNvdPojo(data.getVulnerabilities(), data.getTimestamp()); + } catch (IOException exception) { + throw new CacheException("Unable to read cached data: " + cacheFilePath, exception); + } + } + + private long saveCvesToCache(File targetFile, CveApiJson20 data, MessageDigest md) { + final ObjectMapper objectMapper = getCacheObjectMapper(); + + long uncompressedSize; + try (FileOutputStream fileOutputStream = new FileOutputStream(targetFile); + GZIPOutputStream gzipOutputStream = new GZIPOutputStream(fileOutputStream); + DigestOutputStream digestOutputStream = new DigestOutputStream(gzipOutputStream, md); + CountingOutputStream countingOutputStream = new CountingOutputStream(digestOutputStream)) { + objectMapper.writeValue(countingOutputStream, data); + uncompressedSize = countingOutputStream.getByteCount(); + } catch (IOException ex) { + throw new CacheException("Unable to write cached data: " + targetFile, ex); + } + + return uncompressedSize; + } + + private ObjectMapper getCacheObjectMapper() { final ObjectMapper objectMapper = new ObjectMapper(); objectMapper.registerModule(new JavaTimeModule()); if (isPrettyPrint()) { objectMapper.enable(SerializationFeature.INDENT_OUTPUT); } - final Object2ObjectOpenHashMap> cves = new Object2ObjectOpenHashMap<>(); - populateKeys(cves); + return objectMapper; + } - ZonedDateTime lastModified = downloadAllUpdates(builder, cves); - if (lastModified != null) { - properties.set("lastModifiedDate", lastModified); + private void saveMetaData(File targetFile, long compressedSize, long uncompressedSize, ZonedDateTime lastChanged, + MessageDigest md) { + String checksum = getHex(md.digest()); + try (FileOutputStream fileOutputStream = new FileOutputStream(targetFile); + OutputStreamWriter osw = new OutputStreamWriter(fileOutputStream, StandardCharsets.UTF_8); + PrintWriter writer = new PrintWriter(osw)) { + final String lmd = DateTimeFormatter.ISO_DATE_TIME.format(lastChanged); + writer.println("lastModifiedDate:" + lmd); + writer.println("size:" + uncompressedSize); + writer.println("gzSize:" + compressedSize); + writer.println("sha256:" + checksum); + } catch (IOException ex) { + throw new CacheException("Unable to write cached meta-data: {}" + targetFile.getAbsolutePath(), ex); } - // update cache - // todo - get format and version from API - final String format = "NVD_CVE"; - final String version = "2.0"; - final String prefix = properties.get("prefix", "nvdcve-"); - - for (Object2ObjectMap.Entry> entry : cves - .object2ObjectEntrySet()) { - if (entry.getValue().isEmpty()) { - continue; - } - final File file = new File(properties.getDirectory(), prefix + entry.getKey() + ".json.gz"); - final File meta = new File(properties.getDirectory(), prefix + entry.getKey() + ".meta"); - final Object2ObjectOpenHashMap yearData; - - // Load existing year data if present - if (file.isFile()) { - yearData = new Object2ObjectOpenHashMap<>(); - try (FileInputStream fis = new FileInputStream(file); GZIPInputStream gzis = new GZIPInputStream(fis)) { - CveApiJson20 data = objectMapper.readValue(gzis, CveApiJson20.class); - boolean isYearData = !"modified".equals(entry.getKey()); - // filter the "modified" data to load only the last 7 days of data - data.getVulnerabilities().stream().filter(cve -> isYearData - || ChronoUnit.DAYS.between(cve.getCve().getLastModified(), ZonedDateTime.now()) <= 7) - .forEach(cve -> yearData.put(cve.getCve().getId(), cve)); - } catch (IOException exception) { - throw new CacheException("Unable to read cached data: " + file, exception); - } - yearData.putAll(entry.getValue()); - } else { - yearData = entry.getValue(); - } + } - List vulnerabilities = new ArrayList(yearData.values()); - vulnerabilities.sort((v1, v2) -> { - return v1.getCve().getId().compareTo(v2.getCve().getId()); - }); - ZonedDateTime timestamp; - Optional maxDate = vulnerabilities.stream().map(v -> v.getCve().getLastModified()) - .max(ZonedDateTime::compareTo); - if (maxDate.isPresent()) { - timestamp = maxDate.get(); - } else if (lastModified != null) { - timestamp = lastModified; - } else { - timestamp = ZonedDateTime.now(); - } - properties.set("lastModifiedDate." + entry.getKey(), timestamp); - CveApiJson20 data = new CveApiJson20(vulnerabilities.size(), 0, vulnerabilities.size(), format, version, - timestamp, vulnerabilities); - MessageDigest md; - try { - md = MessageDigest.getInstance("SHA-256"); - } catch (NoSuchAlgorithmException e) { - throw new CacheException("Unable to calculate sha256 checksum", e); - } - long byteCount = 0; - try (FileOutputStream fileOutputStream = new FileOutputStream(file); - GZIPOutputStream gzipOutputStream = new GZIPOutputStream(fileOutputStream); - DigestOutputStream digestOutputStream = new DigestOutputStream(gzipOutputStream, md); - CountingOutputStream countingOutputStream = new CountingOutputStream(digestOutputStream)) { - objectMapper.writeValue(countingOutputStream, data); - byteCount = countingOutputStream.getByteCount(); - } catch (IOException ex) { - throw new CacheException("Unable to write cached data: " + file, ex); - } - String checksum = getHex(md.digest()); - try (FileOutputStream fileOutputStream = new FileOutputStream(meta); - OutputStreamWriter osw = new OutputStreamWriter(fileOutputStream, StandardCharsets.UTF_8); - PrintWriter writer = new PrintWriter(osw)) { - final String lmd = DateTimeFormatter.ISO_DATE_TIME.format(timestamp); - writer.println("lastModifiedDate:" + lmd); - writer.println("size:" + byteCount); - writer.println("gzSize:" + file.length()); - writer.println("sha256:" + checksum); - } catch (IOException ex) { - throw new CacheException("Unable to write cached meta-data: " + file, ex); - } + private ZonedDateTime determineExistingCacheFileLastChanged(Path cacheFilePath) { + File cacheFile = cacheFilePath.toFile(); + if (!cacheFile.exists() || cacheFile.length() == 0) { + return null; } - return 0; + + return ZonedDateTime.ofInstant(Instant.ofEpochMilli(cacheFile.getAbsoluteFile().lastModified()), + ZoneId.systemDefault()); } - private static void populateKeys( - Object2ObjectOpenHashMap> cves) { - cves.put("modified", new Object2ObjectOpenHashMap<>()); - for (int i = 2002; i <= Year.now().getValue(); i++) { - cves.put(Integer.toString(i), new Object2ObjectOpenHashMap<>()); - } + private Path buildCacheTargetFileForYear(CacheProperties properties, int year) { + final String prefix = properties.get("prefix", "nvdcve-"); + return Path.of(properties.getDirectory().getPath(), prefix + year + ".json.gz"); } - private ZonedDateTime downloadAllUpdates(NvdCveClientBuilder builder, - Object2ObjectOpenHashMap> cves) { - ZonedDateTime lastModified = null; - int count = 0; + private Path buildMetaDataTargetFileForYear(CacheProperties properties, int year) { + final String prefix = properties.get("prefix", "nvdcve-"); + return Path.of(properties.getDirectory().getPath(), prefix + year + ".meta"); + } + + /** + * removes all entries that are older than MAX_AGE_OF_MODIFIED_DIFF - operates on the reference to reduce copies + */ + private void removeOutdatedCves(List cves) { + cves.removeIf((item) -> + // remove all items that are older than 8 days + !(ChronoUnit.DAYS.between(item.getCve().getLastModified(), ZonedDateTime.now()) <= MAX_AGE_OF_MODIFIED_DIFF)); + } + + /** + * Fetching from the NVD api, paginated (max entries per page). Aggregates all entries of all pages, sorts them by + * publishing date (ASC) and ensures those are unique. + */ + private CvesNvdPojo fetchFromNVDApi(NvdCveClientBuilder apiBuilder) { // retrieve from NVD API - try (NvdCveClient api = builder.build(); IProgressMonitor monitor = new ProgressMonitor(interactive, "NVD")) { + try (NvdCveClient api = apiBuilder.build(); + IProgressMonitor monitor = new ProgressMonitor(interactive, "NVD")) { Runtime.getRuntime().addShutdownHook(new JlineShutdownHook()); + + // we use a set for de-duplication + Set vulnerabilities = new HashSet<>(); + + // crawl the api page by page while (api.hasNext()) { Collection data = api.next(); - collectCves(cves, data); - lastModified = api.getLastUpdated(); - count += data.size(); - CVE_LOAD_COUNTER.set(count); - monitor.updateProgress("NVD", count, api.getTotalAvailable()); + vulnerabilities.addAll(data); + if (!data.isEmpty()) { + CVE_LOAD_COUNTER.set(data.size()); + monitor.updateProgress("NVD", (int) CVE_LOAD_COUNTER.get(), api.getTotalAvailable()); + } } + CVE_COUNTER.set(vulnerabilities.size()); + + // convert to sorted list by id + List sorted = vulnerabilities.stream().sorted(Comparator.comparing(v -> v.getCve().getId())) + .collect(Collectors.toList()); + return new CvesNvdPojo(sorted, api.getLastUpdated()); } catch (Exception ex) { - LOG.debug("\nERROR", ex); throw new CacheException("Unable to complete NVD cache update due to error: " + ex.getMessage()); } - CVE_COUNTER.set(cves.values().stream().mapToLong(Object2ObjectOpenHashMap::size).sum()); - return lastModified; } /** @@ -412,22 +588,12 @@ public static String getHex(byte[] raw) { return hex.toString(); } - private void collectCves(Object2ObjectOpenHashMap> cves, - Collection vulnerabilities) { - for (DefCveItem item : vulnerabilities) { - cves.get(getNvdYear(item)).put(item.getCve().getId(), item); - if (ChronoUnit.DAYS.between(item.getCve().getLastModified(), ZonedDateTime.now()) <= 7) { - cves.get("modified").put(item.getCve().getId(), item); - } - } - } - - private String getNvdYear(DefCveItem item) { - int year = item.getCve().getPublished().getYear(); - if (year < 2002) { - year = 2002; + private MessageDigest getDigestAlg() { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new CacheException("Unable to calculate sha256 checksum", e); } - return Integer.toString(year); } private int processRequest(NvdCveClientBuilder builder) throws IOException { diff --git a/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/model/CvesNvdPojo.java b/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/model/CvesNvdPojo.java new file mode 100644 index 00000000..8b3812dd --- /dev/null +++ b/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/model/CvesNvdPojo.java @@ -0,0 +1,31 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2025 Jeremy Long. All Rights Reserved. + */ +package io.github.jeremylong.vulnz.cli.model; + +import io.github.jeremylong.openvulnerability.client.nvd.DefCveItem; +import java.time.ZonedDateTime; +import java.util.List; + +public class CvesNvdPojo { + public List vulnerabilities; + public ZonedDateTime lastUpdated; + + public CvesNvdPojo(List vulnerabilities, ZonedDateTime lastUpdated) { + this.vulnerabilities = vulnerabilities; + this.lastUpdated = lastUpdated; + } +}