diff --git a/insights/__init__.py b/insights/__init__.py new file mode 100644 index 000000000..20854f2ad --- /dev/null +++ b/insights/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# diff --git a/insights/apps.py b/insights/apps.py new file mode 100644 index 000000000..7200f28cd --- /dev/null +++ b/insights/apps.py @@ -0,0 +1,14 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from django.apps import AppConfig + + +class InsightsConfig(AppConfig): + name = "insights" diff --git a/insights/graphs/__init__.py b/insights/graphs/__init__.py new file mode 100644 index 000000000..4702f4386 --- /dev/null +++ b/insights/graphs/__init__.py @@ -0,0 +1,89 @@ +from functools import partial + +from insights.models import GraphDefinition + +from .importer_panel import _get_snapshot_data +from .importer_panel import build_importer_exploit_columns +from .importer_panel import build_importer_package_columns +from .importer_panel import collect_importers +from .package_panel import collect_cwes +from .package_panel import collect_ecosystem_dist +from .package_panel import collect_packages +from .package_panel import format_ecosystem_dist +from .package_panel import format_top_cwes +from .package_panel import format_top_packages +from .severity_panel import collect_severities +from .severity_panel import get_severity_snapshot_data + +GRAPHS = { + "pkg-dist-donut": GraphDefinition( + id="pkg-dist-donut", + title="Ecosystem Distribution", + panel="package_panel", + chart_type="donut", + data_fn=format_ecosystem_dist, + collect_fn=collect_ecosystem_dist, + ), + "pkg-name-donut": GraphDefinition( + id="pkg-name-donut", + title="Top 10 Packages", + panel="package_panel", + chart_type="donut", + data_fn=format_top_packages, + collect_fn=collect_packages, + is_per_package=True, + ), + "pkg-cwe-bar": GraphDefinition( + id="pkg-cwe-bar", + title="Top 10 CWE Distribution", + panel="package_panel", + chart_type="colored_bar", + data_fn=format_top_cwes, + collect_fn=collect_cwes, + is_per_package=False, + has_search=True, + ), + "severity-scatter-plot": GraphDefinition( + id="severity-scatter-plot", + title="Severity Distribution across Packages", + panel="severity_panel", + chart_type="scatter", + data_fn=get_severity_snapshot_data, + collect_fn=collect_severities, + has_search=True, + ), + "importer-empty-pkg-bar": GraphDefinition( + id="importer-empty-pkg-bar", + title="Affected Package Coverage across Importers", + panel="importer_panel", + chart_type="importer_bar", + data_fn=partial(_get_snapshot_data, build_columns_fn=build_importer_package_columns), + collect_fn=collect_importers, + ), + "importer-exploit-bar": GraphDefinition( + id="importer-exploit-bar", + title="Exploit Coverage across Importers", + panel="importer_panel", + chart_type="importer_bar", + data_fn=partial(_get_snapshot_data, build_columns_fn=build_importer_exploit_columns), + collect_fn=collect_importers, + ), +} + +PANELS = [ + "overview_panel", + "package_panel", + "severity_panel", + "importer_panel", + "data_quality_panel", +] +PANEL_LABELS = { + "overview_panel": "Overview", + "package_panel": "Package Analytics", + "severity_panel": "Severity Analytics", + "importer_panel": "Importer Analytics", + "data_quality_panel": "Data Quality Analytics", +} +PANEL_LAYOUTS = { + "package_panel": "split_top", +} diff --git a/insights/graphs/importer_panel.py b/insights/graphs/importer_panel.py new file mode 100644 index 000000000..ea68df827 --- /dev/null +++ b/insights/graphs/importer_panel.py @@ -0,0 +1,181 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +from typing import Any +from typing import Dict + +from django.db.models import Count +from django.db.models import Exists +from django.db.models import OuterRef +from django.db.models import Q + +from insights.models import ImporterInsight +from insights.utils import format_importer_name +from vulnerabilities.models import AdvisoryExploit +from vulnerabilities.models import AdvisoryV2 + +# Ignore Phantom Importers that don't collect Affected Packages +IGNORED_IMPORTERS = { + "epss_importer_v2", + "epss", + "vulnrichment_importer_v2", + "suse_importer_v2", +} + + +def collect_importers(pipeline): + """Calculates and stores importer insights.""" + + # Two graphs use this function, so we avoid rerunning the query twice + if pipeline.importer_insights: + return + + kev_exploits = AdvisoryExploit.objects.filter(advisory=OuterRef("pk"), data_source="KEV") + metasploit_exploits = AdvisoryExploit.objects.filter( + advisory=OuterRef("pk"), data_source="Metasploit" + ) + exploitdb_exploits = AdvisoryExploit.objects.filter( + advisory=OuterRef("pk"), data_source="Exploit-DB" + ) + + importer_statistics = ( + AdvisoryV2.objects.filter(is_latest=True) + .exclude(datasource_id__in=IGNORED_IMPORTERS) + .annotate( + has_kev=Exists(kev_exploits), + has_metasploit=Exists(metasploit_exploits), + has_exploitdb=Exists(exploitdb_exploits), + ) + .values("datasource_id") + .annotate( + total_advisories=Count("avid", distinct=True), + advisories_with_packages=Count( + "avid", filter=Q(impacted_packages__affecting_packages__isnull=False), distinct=True + ), + advisories_with_ghost_packages=Count( + "avid", filter=Q(impacted_packages__affecting_packages__is_ghost=True), distinct=True + ), + advisories_with_kev=Count("avid", filter=Q(has_kev=True), distinct=True), + advisories_with_metasploit=Count( + "avid", filter=Q(has_kev=False, has_metasploit=True), distinct=True + ), + advisories_with_exploitdb=Count( + "avid", + filter=Q(has_kev=False, has_metasploit=False, has_exploitdb=True), + distinct=True, + ), + ) + .order_by("-total_advisories") + .iterator() + ) + + for record in importer_statistics: + pipeline.importer_insights.append( + ImporterInsight( + importer=record["datasource_id"], + total_advisories=record["total_advisories"], + advisories_with_packages=record["advisories_with_packages"], + advisories_with_ghost_packages=record["advisories_with_ghost_packages"], + advisories_with_kev=record["advisories_with_kev"], + advisories_with_metasploit=record["advisories_with_metasploit"], + advisories_with_exploitdb=record["advisories_with_exploitdb"], + ) + ) + + +def _get_snapshot_data(snapshot: Any, build_columns_fn) -> Dict[str, Any]: + """Helper to format importer statistics using a provided column builder.""" + all_importers = list(snapshot.importer_insights.order_by("-total_advisories")) + data = {} + + if all_importers: + # Top 5 by default for global view + data["global"] = build_columns_fn(all_importers[:5]) + + # Individual data for each importer + for importer_insight in all_importers: + formatted_name = format_importer_name(importer_insight.importer) + data[formatted_name] = build_columns_fn([importer_insight]) + + return data + + +def build_importer_package_columns(importer_list) -> Dict[str, Any]: + """Helper to build columns array for affected package horizontal bar charts.""" + importers = [] + total_advisories = [] + advisories_with_packages = [] + advisories_without_packages = [] + advisories_with_ghost_packages = [] + advisories_without_ghost_packages = [] + + for importer_insight in importer_list: + importers.append(format_importer_name(importer_insight.importer)) + total_advisories.append(importer_insight.total_advisories) + advisories_with_packages.append(importer_insight.advisories_with_packages) + advisories_without_packages.append( + importer_insight.total_advisories - importer_insight.advisories_with_packages + ) + advisories_with_ghost_packages.append(importer_insight.advisories_with_ghost_packages) + advisories_without_ghost_packages.append( + importer_insight.advisories_with_packages + - importer_insight.advisories_with_ghost_packages + ) + + return { + "x_categories": importers, + "columns": [ + ["total_advisories"] + total_advisories, + ["advisories_without_packages"] + advisories_without_packages, + ["advisories_with_packages"] + advisories_with_packages, + ["advisories_without_ghost_packages"] + advisories_without_ghost_packages, + ["advisories_with_ghost_packages"] + advisories_with_ghost_packages, + ], + "groups": [ + ["advisories_without_packages", "advisories_with_packages"], + ["advisories_without_ghost_packages", "advisories_with_ghost_packages"], + ], + } + + +def build_importer_exploit_columns(importer_list) -> Dict[str, Any]: + """Helper to build columns array for exploit horizontal bar charts.""" + importers = [] + total_advisories = [] + advisories_with_kev = [] + advisories_with_metasploit = [] + advisories_with_exploitdb = [] + advisories_without_exploits = [] + + for importer_insight in importer_list: + importers.append(format_importer_name(importer_insight.importer)) + total_advisories.append(importer_insight.total_advisories) + advisories_with_kev.append(importer_insight.advisories_with_kev) + advisories_with_metasploit.append(importer_insight.advisories_with_metasploit) + advisories_with_exploitdb.append(importer_insight.advisories_with_exploitdb) + + total_with_exploits = ( + importer_insight.advisories_with_kev + + importer_insight.advisories_with_metasploit + + importer_insight.advisories_with_exploitdb + ) + advisories_without_exploits.append(importer_insight.total_advisories - total_with_exploits) + + return { + "x_categories": importers, + "columns": [ + ["total_advisories"] + total_advisories, + ["advisories_without_exploits"] + advisories_without_exploits, + ["advisories_with_exploitdb"] + advisories_with_exploitdb, + ["advisories_with_metasploit"] + advisories_with_metasploit, + ["advisories_with_kev"] + advisories_with_kev, + ], + "groups": [ + ["advisories_with_exploitdb", "advisories_with_metasploit", "advisories_with_kev"] + ], + } diff --git a/insights/graphs/package_panel.py b/insights/graphs/package_panel.py new file mode 100644 index 000000000..1b87a2543 --- /dev/null +++ b/insights/graphs/package_panel.py @@ -0,0 +1,157 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +from collections import Counter +from collections import defaultdict +from typing import Any +from typing import Dict + +from django.db.models import Count +from django.db.models import F + +from insights.models import PackageCWEInsight +from insights.models import PackageInsight +from insights.models import PackageNameInsight +from insights.utils import get_cwe_label +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import PackageV2 + +# Generic OWASP category CWEs used by NVD, not actual weakness IDs. +IGNORED_CWE_IDS = [937, 1035] + + +# Package Distribution +def collect_ecosystem_dist(pipeline: Any) -> None: + """Collect total package counts per ecosystem.""" + package_counts = ( + PackageV2.objects.filter(type__in=pipeline.packages) + .values("type") + .annotate(total_package_count=Count("id")) + .iterator() + ) + + for package_stat in package_counts: + package_type = package_stat["type"] + if package_type in pipeline.package_insights: + pipeline.package_insights[package_type].total_packages = package_stat[ + "total_package_count" + ] + + +def format_ecosystem_dist(snapshot: Any) -> Dict[str, Any]: + """Format the Ecosystem Distribution Graph""" + stats = list(snapshot.package_insights.exclude(package="global").order_by("-total_packages")) + + columns = [[stat.package, stat.total_packages] for stat in stats[:15]] + + others_count = sum(stat.total_packages for stat in stats[15:]) + others_list = [[stat.package, stat.total_packages] for stat in stats[15:25]] + if others_count > 0: + columns.append(["Others", others_count]) + + return {"global": {"columns": columns, "others_list": others_list}} + + +# Top 10 Packages +def collect_packages(pipeline: Any, package_type: str) -> None: + """Collect the top 10 packages for a package type""" + top_10 = ( + PackageV2.objects.filter(type=package_type) + .values("name") + .annotate(count=Count("id")) + .order_by("-count")[:10] + ) + + insight_obj = pipeline.package_insights[package_type] + for stat in top_10: + pipeline.package_names.append( + PackageNameInsight(package_insight=insight_obj, name=stat["name"], count=stat["count"]) + ) + + +def build_name_chart_columns(name_counts: dict) -> Dict[str, Any]: + """Helper to build package name donut charts.""" + return {"columns": [[name, count] for name, count in name_counts.items()]} + + +def format_top_packages(snapshot: Any) -> Dict[str, Any]: + """ + Format name data for the frontend donut chart. + Returns Top 10 names globally and for each package type. + """ + data = {} + global_counts = defaultdict(int) + + for package_insight in ( + snapshot.package_insights.exclude(package="global").prefetch_related("names").all() + ): + name_counts = {ns.name: ns.count for ns in package_insight.names.all()} + data[package_insight.package] = build_name_chart_columns(name_counts) + for name, count in name_counts.items(): + global_counts[name] += count + + top_global = dict(Counter(global_counts).most_common(10)) + data["global"] = build_name_chart_columns(top_global) + return data + + +# Top CWE Distribution +def compute_top_cwes(packages=None) -> dict: + """Computes the top 10 CWEs for the given queryset of packages. If None, computes globally.""" + qs = AdvisoryV2.objects.filter(weaknesses__isnull=False) + if packages is not None: + qs = qs.filter(impacted_packages__affecting_packages__in=packages) + + top_10 = ( + qs.values(cwe=F("weaknesses__cwe_id")) + .exclude(cwe__in=IGNORED_CWE_IDS) + .annotate(count=Count("avid", distinct=True)) + .order_by("-count")[:10] + ) + return {stat["cwe"]: stat["count"] for stat in top_10 if stat["cwe"]} + + +def collect_cwes(pipeline: Any) -> None: + """Collect the global top 10 CWE distribution.""" + cwe_counts = compute_top_cwes() + + if "global" not in pipeline.package_insights: + pipeline.package_insights["global"] = PackageInsight(package="global") + + insight_obj = pipeline.package_insights["global"] + for cwe_id, count in cwe_counts.items(): + pipeline.package_cwes.append( + PackageCWEInsight(package_insight=insight_obj, cwe_id=cwe_id, count=count) + ) + + +def build_cwe_chart_columns(cwe_counts: dict) -> Dict[str, Any]: + """Helper to CWE chart.""" + # Sort CWEs by count in descending order + sorted_cwes = sorted(cwe_counts.items(), key=lambda item: item[1], reverse=True) + return { + "columns": [ + ["x"] + [cwe_id for cwe_id, count in sorted_cwes], + ["Vulnerabilities"] + [count for cwe_id, count in sorted_cwes], + ], + "full_labels": [get_cwe_label(cwe_id) for cwe_id, count in sorted_cwes], + } + + +def format_top_cwes(snapshot: Any) -> Dict[str, Any]: + """Format CWE distribution data.""" + data = {} + try: + global_insight = snapshot.package_insights.get(package="global") + cwe_counts = {cwe.cwe_id: cwe.count for cwe in global_insight.cwes.all()} + if cwe_counts: + data["global"] = build_cwe_chart_columns(cwe_counts) + except PackageInsight.DoesNotExist: + pass + + return data diff --git a/insights/graphs/severity_panel.py b/insights/graphs/severity_panel.py new file mode 100644 index 000000000..c386492ca --- /dev/null +++ b/insights/graphs/severity_panel.py @@ -0,0 +1,68 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +from typing import Any +from typing import Dict + +from django.db.models import Count + +from insights.models import SeverityInsight +from vulnerabilities.models import AdvisoryV2 + +CVSS_SCORE_LABELS = ["0-1", "1-2", "2-3", "3-4", "4-5", "5-6", "6-7", "7-8", "8-9", "9-10"] + + +def aggregate_severity_buckets(queryset: Any, count_field: str) -> list[int]: + """Helper to aggregate a queryset of severity values into exactly 10 CVSS buckets.""" + buckets = [0] * 10 + + for severity in queryset: + try: + score = float(severity.get("severities__value")) + if 0 <= score <= 10: + buckets[min(9, int(score))] += severity[count_field] + except (ValueError, TypeError): + continue + + return buckets + + +def collect_severities(pipeline: Any) -> None: + """Pre-compute the global severity distribution.""" + advisory_severities = ( + AdvisoryV2.objects.filter( + severities__isnull=False, severities__scoring_system__icontains="cvss" + ) + .values("severities__value") + .annotate(advisory_count=Count("avid", distinct=True)) + .iterator() + ) + + severity_buckets = aggregate_severity_buckets(advisory_severities, "advisory_count") + pipeline.severity_insight = SeverityInsight(buckets=severity_buckets) + + +def format_severity_chart_data(buckets: list[int]) -> Dict[str, Any]: + """Formats a list of 10 bucket values into the structure required by billboard.js.""" + return { + "columns": [ + ["x"] + CVSS_SCORE_LABELS, + ["Vulnerabilities"] + buckets, + ] + } + + +def get_severity_snapshot_data(snapshot: Any) -> Dict[str, Any]: + """Return the global severity distribution for the frontend scatter plot.""" + if hasattr(snapshot, "severity_insight"): + insight = snapshot.severity_insight + global_buckets = insight.buckets + else: + global_buckets = [0] * 10 + + return {"global": format_severity_chart_data(global_buckets)} diff --git a/insights/insights_snapshot_pipeline.py b/insights/insights_snapshot_pipeline.py new file mode 100644 index 000000000..2d9261623 --- /dev/null +++ b/insights/insights_snapshot_pipeline.py @@ -0,0 +1,90 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +from aboutcode.pipeline import LoopProgress +from django.db import transaction + +from insights.graphs import GRAPHS +from insights.models import DailySnapshot +from insights.models import ImporterInsight +from insights.models import PackageCWEInsight +from insights.models import PackageInsight +from insights.models import PackageNameInsight +from vulnerabilities.models import PackageV2 +from vulnerabilities.pipelines import VulnerableCodePipeline + + +class InsightsSnapshotPipeline(VulnerableCodePipeline): + """Pipeline to compute aggregated statistics for the Insights Dashboard.""" + + pipeline_id = "insights_snapshot" + + @classmethod + def steps(cls): + return ( + cls.compute_graph_analytics, + cls.save_snapshot, + ) + + def compute_graph_analytics(self): + """Run graph collect_fns to compute analytics.""" + + # List all package types + self.packages = list(PackageV2.objects.order_by().values_list("type", flat=True).distinct()) + + self.package_insights = {pkg: PackageInsight(package=pkg) for pkg in self.packages} + self.package_names = [] + self.package_cwes = [] + self.importer_insights = [] + self.severity_insight = None + + active_graphs = { + graph_id: graph_def for graph_id, graph_def in GRAPHS.items() if graph_def.collect_fn + } + + # Count steps for progress bar + total_steps = sum( + len(self.packages) if graph_def.is_per_package else 1 + for graph_def in active_graphs.values() + ) + + progress = LoopProgress(total_iterations=total_steps, logger=self.log, progress_step=1) + progress_iter = iter(progress.iter(range(total_steps))) + + for graph_id, graph_def in active_graphs.items(): + self.log(f"Running collect_fn for {graph_id}") + + if graph_def.is_per_package: + for pkg in self.packages: + next(progress_iter, None) + graph_def.collect_fn(self, pkg) + else: + next(progress_iter, None) + graph_def.collect_fn(self) + + @transaction.atomic + def save_snapshot(self): + self.log("Saving snapshot") + snapshot = DailySnapshot.objects.create() + + for insight in self.package_insights.values(): + insight.snapshot = snapshot + + for insight in self.importer_insights: + insight.snapshot = snapshot + + if self.severity_insight: + self.severity_insight.snapshot = snapshot + self.severity_insight.save() + + PackageInsight.objects.bulk_create(self.package_insights.values(), batch_size=5000) + PackageNameInsight.objects.bulk_create(self.package_names, batch_size=5000) + PackageCWEInsight.objects.bulk_create(self.package_cwes, batch_size=5000) + ImporterInsight.objects.bulk_create(self.importer_insights, batch_size=5000) + + self.log("Snapshot saved successfully.") diff --git a/insights/migrations/0001_initial.py b/insights/migrations/0001_initial.py new file mode 100644 index 000000000..26b349636 --- /dev/null +++ b/insights/migrations/0001_initial.py @@ -0,0 +1,174 @@ +# Generated by Django 5.2.11 on 2026-07-23 21:56 + +import django.contrib.postgres.fields +import django.db.models.deletion +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="DailySnapshot", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("created_at", models.DateTimeField(auto_now_add=True)), + ], + options={ + "ordering": ["-created_at"], + "get_latest_by": "created_at", + }, + ), + migrations.CreateModel( + name="PackageInsight", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("package", models.CharField(max_length=50)), + ("total_packages", models.IntegerField(default=0)), + ( + "snapshot", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="package_insights", + to="insights.dailysnapshot", + ), + ), + ], + ), + migrations.CreateModel( + name="PackageCWEInsight", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("cwe_id", models.CharField(max_length=50)), + ("count", models.IntegerField()), + ( + "package_insight", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="cwes", + to="insights.packageinsight", + ), + ), + ], + ), + migrations.CreateModel( + name="PackageNameInsight", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("name", models.CharField(max_length=255)), + ("count", models.IntegerField()), + ( + "package_insight", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="names", + to="insights.packageinsight", + ), + ), + ], + ), + migrations.CreateModel( + name="SeverityInsight", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ( + "buckets", + django.contrib.postgres.fields.ArrayField( + base_field=models.IntegerField(), + default=list, + help_text="Scores mapped to buckets 0-10 (e.g. 0.0-0.9 -> index 0)", + size=10, + ), + ), + ( + "snapshot", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + related_name="severity_insight", + to="insights.dailysnapshot", + ), + ), + ], + ), + migrations.CreateModel( + name="ImporterInsight", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("importer", models.CharField(max_length=100)), + ("total_advisories", models.IntegerField(default=0)), + ("advisories_with_packages", models.IntegerField(default=0)), + ("advisories_with_ghost_packages", models.IntegerField(default=0)), + ("advisories_with_kev", models.IntegerField(default=0)), + ("advisories_with_metasploit", models.IntegerField(default=0)), + ("advisories_with_exploitdb", models.IntegerField(default=0)), + ( + "snapshot", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="importer_insights", + to="insights.dailysnapshot", + ), + ), + ], + options={ + "constraints": [ + models.UniqueConstraint( + fields=("snapshot", "importer"), name="unique_snapshot_importer" + ) + ], + }, + ), + migrations.AddConstraint( + model_name="packageinsight", + constraint=models.UniqueConstraint( + fields=("snapshot", "package"), name="unique_snapshot_package" + ), + ), + migrations.AddConstraint( + model_name="packagecweinsight", + constraint=models.UniqueConstraint( + fields=("package_insight", "cwe_id"), name="unique_package_cwe" + ), + ), + migrations.AddConstraint( + model_name="packagenameinsight", + constraint=models.UniqueConstraint( + fields=("package_insight", "name"), name="unique_package_name" + ), + ), + ] diff --git a/insights/migrations/__init__.py b/insights/migrations/__init__.py new file mode 100644 index 000000000..20854f2ad --- /dev/null +++ b/insights/migrations/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# diff --git a/insights/models.py b/insights/models.py new file mode 100644 index 000000000..ef19771b0 --- /dev/null +++ b/insights/models.py @@ -0,0 +1,108 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +from dataclasses import dataclass +from typing import Any +from typing import Callable +from typing import Dict +from typing import Optional + +from django.contrib.postgres.fields import ArrayField +from django.db import models + + +class DailySnapshot(models.Model): + created_at = models.DateTimeField(auto_now_add=True) + + class Meta: + get_latest_by = "created_at" + ordering = ["-created_at"] + + +class PackageInsight(models.Model): + snapshot = models.ForeignKey( + DailySnapshot, related_name="package_insights", on_delete=models.CASCADE + ) + package = models.CharField(max_length=50) + total_packages = models.IntegerField(default=0) + + class Meta: + constraints = [ + models.UniqueConstraint(fields=["snapshot", "package"], name="unique_snapshot_package") + ] + + +class PackageNameInsight(models.Model): + package_insight = models.ForeignKey( + PackageInsight, related_name="names", on_delete=models.CASCADE + ) + name = models.CharField(max_length=255) + count = models.IntegerField() + + class Meta: + constraints = [ + models.UniqueConstraint(fields=["package_insight", "name"], name="unique_package_name") + ] + + +class PackageCWEInsight(models.Model): + package_insight = models.ForeignKey( + PackageInsight, related_name="cwes", on_delete=models.CASCADE + ) + cwe_id = models.CharField(max_length=50) + count = models.IntegerField() + + class Meta: + constraints = [ + models.UniqueConstraint(fields=["package_insight", "cwe_id"], name="unique_package_cwe") + ] + + +class SeverityInsight(models.Model): + snapshot = models.OneToOneField( + DailySnapshot, related_name="severity_insight", on_delete=models.CASCADE + ) + buckets = ArrayField( + models.IntegerField(), + size=10, + default=list, + help_text="Scores mapped to buckets 0-10 (e.g. 0.0-0.9 -> index 0)", + ) + + +class ImporterInsight(models.Model): + snapshot = models.ForeignKey( + DailySnapshot, related_name="importer_insights", on_delete=models.CASCADE + ) + importer = models.CharField(max_length=100) + total_advisories = models.IntegerField(default=0) + advisories_with_packages = models.IntegerField(default=0) + advisories_with_ghost_packages = models.IntegerField(default=0) + advisories_with_kev = models.IntegerField(default=0) + advisories_with_metasploit = models.IntegerField(default=0) + advisories_with_exploitdb = models.IntegerField(default=0) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["snapshot", "importer"], name="unique_snapshot_importer" + ) + ] + + +@dataclass(frozen=True) +class GraphDefinition: + + id: str + title: str + panel: str + chart_type: str + data_fn: Callable[[Dict[str, Any]], Dict[str, Any]] + collect_fn: Optional[Callable] = None + is_per_package: bool = False + has_search: bool = False diff --git a/insights/static/insights/css/insights.css b/insights/static/insights/css/insights.css new file mode 100644 index 000000000..6973d7b26 --- /dev/null +++ b/insights/static/insights/css/insights.css @@ -0,0 +1,301 @@ +/* Palette and CSS variables */ +:root { + --insights-palette-0: #3273dc; + --insights-palette-1: #00d1b2; + --insights-palette-2: #ff3860; + --insights-palette-3: #ffdd57; + --insights-palette-4: #48c774; + --insights-palette-5: #8361b1; + --insights-palette-6: #ff6b35; + --insights-palette-7: #209cee; + --insights-palette-8: #e84393; + --insights-palette-9: #d12323bf; +} + +/* Dashboard layout */ +.insights-layout { + display: flex; + align-items: flex-start; + gap: 1.5rem; + padding: 1.5rem 1rem; +} + +.insights-sidenav { + width: 220px; + flex-shrink: 0; + position: sticky; + top: 5rem; + height: calc(100vh - 7rem); + display: flex; + flex-direction: column; +} + +.insights-sidenav nav.menu { + flex: 1; + overflow-y: auto; + min-height: 0; + margin-bottom: 1rem; +} + +.insights-sidenav .menu-label { + color: #000; + font-weight: 600; +} + +.insights-sidenav .menu-list { + list-style: none !important; + padding-left: 0; + margin-left: 0; +} + +.insights-sidenav .menu-list li { + margin-bottom: 0.25rem; +} + +.insights-sidenav .menu-list a { + color: #000; + padding-left: 1rem; + border-radius: 0 4px 4px 0; +} + +.insights-sidenav .menu-list a:hover { + background-color: #f6f8fa; + color: #000; +} + +.insights-sidenav .menu-list a.is-active { + background-color: #f0f5fa; + color: var(--insights-palette-0); + font-weight: 600; + border-left: 3px solid var(--insights-palette-0); + padding-left: calc(1rem - 3px); +} + +.insights-main { + flex: 1; + min-width: 0; +} + +.graph-inner { + max-width: 860px; + margin-left: auto; + margin-right: auto; +} + +.graph-row { + margin-bottom: 1.25rem; + padding: 1.25rem 1.5rem; + background: #fff; +} + +.graph-row h3 { + margin-bottom: 0.75rem; + font-size: 1rem; + font-weight: 600; + color: #2c3e50; +} + + + +/* Chart containers and loading state */ +.bb svg { + width: 100% !important; +} + +.chart-container { + width: 100%; + min-height: 320px; +} + +.chart-container.is-pie { + min-height: 420px; +} + +.chart-loading { + display: flex; + align-items: center; + justify-content: center; + min-height: 200px; + color: #7a7a7a; + font-size: 0.9rem; +} + + + +/* Billboard.js tooltip overrides */ +.bb-tooltip-container table.bb-tooltip { + min-width: 260px; + white-space: nowrap; +} + +.bb-tooltip-container table.bb-tooltip th { + padding: 0.4rem 0.8rem !important; +} + +.bb-tooltip-container table.bb-tooltip td { + padding: 0.3rem 0.8rem !important; +} + +.bb-tooltip-container table.bb-tooltip td.value, +.bb-tooltip-container table.bb-tooltip td:last-child { + text-align: right; + font-variant-numeric: tabular-nums; + font-weight: 600; + padding-right: 1.2rem !important; +} + +.sidenav-footer { + margin-top: auto; + padding: 1rem 0; + border-top: 1px solid #e8e8e8; +} + +.snapshot-text { + font-size: 0.75rem; + color: #000; + line-height: 1.4; +} + +.snapshot-text strong { + color: #000; +} + +/* Classes replacing inline styles */ +.panel-section { + background-color: #fff; + overflow: hidden; +} + +.flat-panel { + margin-bottom: 0; + box-shadow: none; +} + +.flat-panel-heading { + border-radius: 0; + display: flex; + justify-content: space-between; + align-items: center; +} + +.w-full { + max-width: 100%; +} + +.card-graph-row { + border: 1px solid #e8e8e8; + border-radius: 6px; + box-shadow: 0 1px 2px rgba(0, 0, 0, 0.02); +} + +.card-graph-title { + border-bottom: 1px solid #e8e8e8; + padding-bottom: 0.75rem; + margin-bottom: 1.25rem; +} + +/* Package Panel Specific Styles */ +.chart-dropdown-wrapper { + display: flex; + justify-content: flex-end; + margin-bottom: 8px; +} + +.severity-chart-layout { + display: flex; + align-items: flex-start; + gap: 0; + width: 100%; + font-family: inherit; +} + +.severity-table-wrapper { + flex: 0 0 380px; + min-width: 260px; +} + +.severity-table { + width: 100%; + border-collapse: collapse; + font-size: 13px; +} + +.sev-th-left { + text-align: left; + padding: 4px 8px; + color: #555; + font-weight: 600; + border-bottom: 1px solid #ddd; +} + +.sev-th-right { + text-align: right; + padding: 4px 8px; + color: #555; + font-weight: 600; + border-bottom: 1px solid #ddd; +} + +.sev-tfoot-tr { + border-top: 1px solid #ddd; +} + +.sev-td-total-label { + padding: 5px 8px; + font-weight: 600; + color: #333; +} + +.sev-td-total-value { + padding: 5px 8px; + text-align: right; + font-weight: 600; + color: #2563eb; +} + +.severity-bb-container { + flex: 1; + min-height: 280px; + margin-top: 52px; +} + +.sev-td-bucket { + padding: 4px 8px; + vertical-align: middle; +} + +.sev-bucket-label { + display: inline-block; + width: 2.5em; + color: #444; +} + +.sev-bucket-bar { + display: inline-block; + height: 7px; + border-radius: 2px; + vertical-align: middle; + margin-left: 4px; +} + +.sev-td-count { + padding: 4px 8px; + text-align: right; + color: #2563eb; +} + +.sev-empty { + color: #aaa; +} + +.card-graph-title-with-search { + display: flex; + justify-content: space-between; + align-items: center; +} + +.severity-search-wrapper { + display: flex; + align-items: center; + margin-left: auto; +} \ No newline at end of file diff --git a/insights/static/insights/js/core.js b/insights/static/insights/js/core.js new file mode 100644 index 000000000..3b046d6e9 --- /dev/null +++ b/insights/static/insights/js/core.js @@ -0,0 +1,57 @@ +// +// Copyright (c) nexB Inc. and others. All rights reserved. +// VulnerableCode is a trademark of nexB Inc. +// SPDX-License-Identifier: Apache-2.0 +// See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +// See https://github.com/aboutcode-org/vulnerablecode for support or download. +// See https://aboutcode.org for more information about nexB OSS projects. +// + +import { renderers } from './renderers.js'; + +export function renderChartWithData(chartId, key, chartDataMap = {}) { + // Handle dropdown charts with key + const chartData = key && chartDataMap[key] ? chartDataMap[key] : chartDataMap; + const chartContainer = document.getElementById(`chart-${chartId}`); + if (!chartContainer) return; + + if (!chartData?.columns?.length) { + chartContainer.innerHTML = "

No data available.

"; + return; + } + + const type = chartContainer.dataset.chartType; + if (renderers[type]) renderers[type](chartId, chartData); +} + +export function initDropdownChart(chartId, chartData, defaultLabel) { + const chartContainer = document.getElementById(`chart-${chartId}`); + if (!chartData || !chartContainer || chartContainer.previousElementSibling?.classList.contains("chart-dropdown-wrapper")) return; + + const options = Object.keys(chartData).filter(k => k !== "global"); + if (options.length) { + chartContainer.insertAdjacentHTML("beforebegin", ` +
+
+ +
+
+ `); + chartContainer.previousElementSibling.querySelector("select").addEventListener("change", ev => + renderChartWithData(chartId, ev.target.value, chartData) + ); + } + renderChartWithData(chartId, "global", chartData); +} + +export function initDashboard() { + const layout = document.querySelector(".insights-layout"); + if (!layout) return; + const data = JSON.parse(document.getElementById("insights-snapshot-data")?.textContent || "{}"); + document.dispatchEvent(new CustomEvent('insightsDataLoaded', { detail: { data, snapshotData: data.snapshot_data || {}, panelId: layout.dataset.panel } })); +} + +setTimeout(initDashboard, 0); diff --git a/insights/static/insights/js/importer_panel.js b/insights/static/insights/js/importer_panel.js new file mode 100644 index 000000000..4840e23a2 --- /dev/null +++ b/insights/static/insights/js/importer_panel.js @@ -0,0 +1,19 @@ +// +// Copyright (c) nexB Inc. and others. All rights reserved. +// VulnerableCode is a trademark of nexB Inc. +// SPDX-License-Identifier: Apache-2.0 +// See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +// See https://github.com/aboutcode-org/vulnerablecode for support or download. +// See https://aboutcode.org for more information about nexB OSS projects. +// + +import { initDropdownChart } from './core.js'; + +document.addEventListener('insightsDataLoaded', (e) => { + if (e.detail.panelId !== "importer_panel") return; + const { snapshotData } = e.detail; + + ["importer-empty-pkg-bar", "importer-exploit-bar"].forEach(id => { + initDropdownChart(id, snapshotData[id], "Top 5 Importers"); + }); +}); diff --git a/insights/static/insights/js/package_panel.js b/insights/static/insights/js/package_panel.js new file mode 100644 index 000000000..fb32ed63b --- /dev/null +++ b/insights/static/insights/js/package_panel.js @@ -0,0 +1,28 @@ +// +// Copyright (c) nexB Inc. and others. All rights reserved. +// VulnerableCode is a trademark of nexB Inc. +// SPDX-License-Identifier: Apache-2.0 +// See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +// See https://github.com/aboutcode-org/vulnerablecode for support or download. +// See https://aboutcode.org for more information about nexB OSS projects. +// + +import { renderChartWithData, initDropdownChart } from './core.js'; + +document.addEventListener('insightsDataLoaded', (e) => { + if (e.detail.panelId !== "package_panel") return; + + const { snapshotData } = e.detail; + const searchData = document.getElementById("graph-search-data")?.textContent; + const parsedSearch = searchData ? JSON.parse(searchData) : null; + + initDropdownChart("pkg-name-donut", snapshotData["pkg-name-donut"], "All Packages"); + + renderChartWithData( + "pkg-cwe-bar", + parsedSearch ? null : "global", + parsedSearch?.["pkg-cwe-bar"] || snapshotData["pkg-cwe-bar"] + ); + + renderChartWithData("pkg-dist-donut", "global", snapshotData["pkg-dist-donut"]); +}); diff --git a/insights/static/insights/js/renderers.js b/insights/static/insights/js/renderers.js new file mode 100644 index 000000000..4b083770d --- /dev/null +++ b/insights/static/insights/js/renderers.js @@ -0,0 +1,160 @@ +// +// Copyright (c) nexB Inc. and others. All rights reserved. +// VulnerableCode is a trademark of nexB Inc. +// SPDX-License-Identifier: Apache-2.0 +// See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +// See https://github.com/aboutcode-org/vulnerablecode for support or download. +// See https://aboutcode.org for more information about nexB OSS projects. +// + +const getPalette = () => Array.from({ length: 10 }, (_, i) => + getComputedStyle(document.documentElement).getPropertyValue(`--insights-palette-${i}`).trim() +).filter(Boolean); + +const BUCKET_COLORS = [ + "#00bb00", "#33bb00", "#66bb00", "#99bb00", "#ccbb00", + "#ffbb00", "#ff9900", "#ff6600", "#ff3300", "#dd0000" +]; + +const formatWholeNumbersOnly = x => Number.isInteger(x) ? x : ""; + +export const renderers = { + donut(id, config) { + const bbConfig = { + bindto: `#chart-${id}`, + data: { columns: config.columns, type: "donut", colors: { "Others": "#a0a0a0" } }, + color: { pattern: getPalette() }, + legend: { show: true } + }; + + if (config.others_list?.length) { + bbConfig.tooltip = { + contents(d, defaultTitle, defaultVal, color) { + if (d[0].id !== "Others") return this.internal.getTooltipContent(d, defaultTitle, defaultVal, color); + + //Customize tooltip to show table for Others + const total = config.columns.reduce((sum, col) => sum + col[1], 0); + let html = ""; + config.others_list.forEach(([name, val]) => { + html += ``; + }); + return html + "
Others
${name}${val.toLocaleString()} (${((val / total) * 100).toFixed(1)}%)
"; + } + }; + } + bb.generate(bbConfig); + }, + + colored_bar(id, config) { + const monoColor = getPalette()[0] || "#3273dc"; + bb.generate({ + bindto: `#chart-${id}`, + data: { x: "x", columns: config.columns, type: "bar", color: () => monoColor }, + axis: { + rotated: true, + x: { type: "category", label: { text: "CWE", position: "outer-middle" } }, + y: { label: { text: "Vulnerabilities", position: "outer-center" }, tick: { format: formatWholeNumbersOnly } } + }, + tooltip: { format: { title: x => config.full_labels?.[x] || x, value: val => val.toLocaleString() } }, + legend: { show: false } + }); + }, + + importer_bar(id, config) { + bb.generate({ + bindto: `#chart-${id}`, + data: { + x: "x", + columns: [["x", ...config.x_categories], ...config.columns], + type: "bar", + colors: { + total_advisories: "#3273dc", advisories_with_packages: "#00d1b2", + advisories_without_packages: "#e74c3c", advisories_with_exploits: "#55d787", + advisories_without_exploits: "#e74c3c", advisories_with_kev: "#ff8c00", + advisories_with_metasploit: "#870eeaff", advisories_with_exploitdb: "#00e676", + advisories_with_ghost_packages: "#db4be9ff", advisories_without_ghost_packages: "#ffd700" + }, + names: { + total_advisories: "All Advisories", advisories_with_packages: "Advisories with a Package", + advisories_without_packages: "Advisories without a Package", advisories_with_exploits: "Advisories with an Exploit", + advisories_without_exploits: "Advisories without an Exploit", advisories_with_kev: "Advisories with KEV", + advisories_with_metasploit: "Advisories with Metasploit", advisories_with_exploitdb: "Advisories with Exploit-DB", + advisories_with_ghost_packages: "Advisories with a Ghost Package", advisories_without_ghost_packages: "Advisories without a Ghost Package" + }, + groups: config.groups + }, + axis: { rotated: true, x: { type: "category" }, y: { tick: { format: formatWholeNumbersOnly } } }, + bar: { width: { ratio: 0.8 } }, tooltip: { grouped: true }, legend: { show: true } + }); + }, + + scatter(id, config) { + const [, ...buckets] = config.columns[0]; + const [, ...counts] = config.columns[1]; + + const rows = document.getElementById(`chart-${id}-rows`); + const total = document.getElementById(`chart-${id}-total`); + const rowTemplate = document.getElementById(`chart-${id}-row-template`); + + if (!rows || !total || !rowTemplate) return; + + const maxCount = Math.max(1, ...counts); + const frag = document.createDocumentFragment(); + + // Build the table rows for each CVSS bucket + buckets.forEach((bucket, i) => { + const count = counts[i]; + const clone = rowTemplate.content.cloneNode(true); + + // Set bucket label (e.g. "9-10") + const labelNode = clone.querySelector('.sev-bucket-label'); + labelNode.textContent = bucket; + + // Calculate size and paint the colored bubble + const barNode = clone.querySelector('.sev-bucket-bar'); + const barWidth = Math.max(2, (count / maxCount) * 140); + barNode.style.width = `${barWidth}px`; + barNode.style.background = BUCKET_COLORS[i]; + + // Display the vulnerability count on tooltip + const countNode = clone.querySelector('.sev-td-count'); + countNode.title = `CVSS ${bucket}: ${count.toLocaleString()}`; + countNode.innerHTML = count.toLocaleString(); + + frag.appendChild(clone); + }); + + // Render the completed table to the DOM + rows.innerHTML = ""; + rows.appendChild(frag); + const sumOfCounts = counts.reduce((sum, count) => sum + count, 0); + total.textContent = sumOfCounts.toLocaleString(); + + bb.generate({ + bindto: `#chart-${id}-bb`, + data: { + x: "x", columns: config.columns, type: "bubble", + color: (defaultColor, dataPoint) => dataPoint.x !== undefined ? BUCKET_COLORS[dataPoint.x] || defaultColor : defaultColor, + labels: false + }, + bubble: { maxR: 55 }, + axis: { + x: { + type: "category", + tick: { multiline: false }, + label: { text: "CVSS Score Range", position: "outer-center" } + }, + y: { show: false, min: 0, max: maxCount * 1.2, padding: { top: 70, bottom: 0 } } + }, + grid: { x: { show: true } }, + legend: { show: false }, + tooltip: { + format: { + title: x => `CVSS ${buckets[x]}`, + name: () => "Vulnerabilities", + value: val => val.toLocaleString() + } + } + }); + } +}; diff --git a/insights/static/insights/js/severity_panel.js b/insights/static/insights/js/severity_panel.js new file mode 100644 index 000000000..1ea345177 --- /dev/null +++ b/insights/static/insights/js/severity_panel.js @@ -0,0 +1,23 @@ +// +// Copyright (c) nexB Inc. and others. All rights reserved. +// VulnerableCode is a trademark of nexB Inc. +// SPDX-License-Identifier: Apache-2.0 +// See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +// See https://github.com/aboutcode-org/vulnerablecode for support or download. +// See https://aboutcode.org for more information about nexB OSS projects. +// + +import { renderChartWithData } from './core.js'; + +document.addEventListener('insightsDataLoaded', (e) => { + if (e.detail.panelId !== "severity_panel") return; + + const chartId = "severity-scatter-plot"; + const searchData = document.getElementById("graph-search-data")?.textContent; + + if (searchData) { + renderChartWithData(chartId, null, JSON.parse(searchData)); + } else { + renderChartWithData(chartId, "global", e.detail.snapshotData[chartId]); + } +}); diff --git a/insights/templates/insights/components/graph_card.html b/insights/templates/insights/components/graph_card.html new file mode 100644 index 000000000..1f0bd4513 --- /dev/null +++ b/insights/templates/insights/components/graph_card.html @@ -0,0 +1,45 @@ +
+

+ {{ graph.title }} + {% if graph.has_search %} +
+ {% if not search_query %} + Showing All Packages + {% else %} + Show All Packages + {% endif %} + +
+
+
+ +
+
+ +
+
+ {% if search_error %} +

{{ search_error }}

+ {% endif %} +
+
+ {% endif %} +

+ + + {% if graph.chart_type == 'line' %} +
+
+ {% endif %} +
diff --git a/insights/templates/insights/components/package_panel_donuts.html b/insights/templates/insights/components/package_panel_donuts.html new file mode 100644 index 000000000..3cfd7ce32 --- /dev/null +++ b/insights/templates/insights/components/package_panel_donuts.html @@ -0,0 +1,12 @@ +
+
+
+

{{ panel.graphs.0.title }}

+
+
+
+

{{ panel.graphs.1.title }}

+
+
+
+
diff --git a/insights/templates/insights/components/severity_scatter_chart.html b/insights/templates/insights/components/severity_scatter_chart.html new file mode 100644 index 000000000..d5de3838a --- /dev/null +++ b/insights/templates/insights/components/severity_scatter_chart.html @@ -0,0 +1,30 @@ +
+
+ + + + + + + + + + + + + + + +
CVSS Score RangeVulnerabilities
Total
+ +
+
+
diff --git a/insights/templates/insights/dashboard.html b/insights/templates/insights/dashboard.html new file mode 100644 index 000000000..165023a49 --- /dev/null +++ b/insights/templates/insights/dashboard.html @@ -0,0 +1,74 @@ +{% extends "base.html" %} +{% load static %} + +{% block title %}Insights{% endblock %} + +{% block extrahead %} + + +{% endblock %} + +{% block content %} +
+ + + +
+ {% for panel in panels %} + {% if panel.id == current_panel_id %} +
+
+
+ {{ panel.label }} +
+
+ +
+ {% if panel.layout == 'split_top' %} + {% include 'insights/components/package_panel_donuts.html' with panel=panel %} + {% for graph in panel.graphs|slice:"2:" %} + {% include 'insights/components/graph_card.html' with graph=graph current_panel_id=current_panel_id search_query=search_query search_error=search_error %} + {% endfor %} + {% else %} + {% for graph in panel.graphs %} + {% include 'insights/components/graph_card.html' with graph=graph current_panel_id=current_panel_id search_query=search_query search_error=search_error %} + {% endfor %} + {% endif %} +
+
+ {% endif %} + {% endfor %} +
+
+{% endblock %} + +{% block scripts %} + + +{{ snapshot_dict|json_script:"insights-snapshot-data" }} +{% if search_results_dict %} +{{ search_results_dict|json_script:"graph-search-data" }} +{% endif %} + + + + +{% endblock %} \ No newline at end of file diff --git a/insights/urls.py b/insights/urls.py new file mode 100644 index 000000000..0b7a9c4e5 --- /dev/null +++ b/insights/urls.py @@ -0,0 +1,17 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from django.urls import path + +from insights import views + +urlpatterns = [ + path("", views.insights_dashboard, name="insights-dashboard"), + path("/", views.insights_dashboard, name="insights-panel"), +] diff --git a/insights/utils.py b/insights/utils.py new file mode 100644 index 000000000..e4d0e6471 --- /dev/null +++ b/insights/utils.py @@ -0,0 +1,30 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +from vulnerabilities.models import Weakness + + +def get_cwe_label(cwe_id: str) -> str: + num = str(cwe_id).replace("CWE-", "") + + try: + name = Weakness().get_cwe(num).name + return f"CWE-{num} · {name}" if name else f"CWE-{num}" + except KeyError: + return f"CWE-{num}" + + +def format_importer_name(name: str) -> str: + """ + Format importer internal names into human-readable labels. + e.g., 'pysec_importer_v2' -> 'Pysec Importer' + """ + parts = name.rsplit("_v", 1) + if len(parts) == 2 and parts[1].isdigit(): + name = parts[0] + return name.replace("_", " ") diff --git a/insights/views.py b/insights/views.py new file mode 100644 index 000000000..5a6f27285 --- /dev/null +++ b/insights/views.py @@ -0,0 +1,113 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +from django.db.models import Count +from django.shortcuts import redirect +from django.shortcuts import render + +from insights.graphs import GRAPHS +from insights.graphs import PANEL_LABELS +from insights.graphs import PANEL_LAYOUTS +from insights.graphs import PANELS +from insights.graphs.package_panel import build_cwe_chart_columns +from insights.graphs.package_panel import compute_top_cwes +from insights.graphs.severity_panel import aggregate_severity_buckets +from insights.graphs.severity_panel import format_severity_chart_data +from insights.models import DailySnapshot +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import PackageV2 + + +def _severity_search(search_query): + """Executes a severity search for a given PUrl""" + packages = PackageV2.objects.search(search_query) + error = None + + if packages.exists(): + severities = ( + AdvisoryV2.objects.filter( + impacted_packages__affecting_packages__in=packages, + severities__isnull=False, + severities__scoring_system__icontains="cvss", + ) + .values("severities__value") + .annotate(sev_count=Count("id", distinct=True)) + ) + buckets = aggregate_severity_buckets(severities, "sev_count") + else: + buckets = [0] * 10 + error = "No data available for this query." + + return format_severity_chart_data(buckets), error + + +def _cwe_search(search_query): + """Executes a CWE search for a given PUrl""" + packages = PackageV2.objects.search(search_query) + error = None + + if packages.exists(): + cwe_counts = compute_top_cwes(packages) + chart_data = build_cwe_chart_columns(cwe_counts) + else: + chart_data = {} + error = "No packages found for this query." + + return {"pkg-cwe-bar": chart_data}, error + + +def insights_dashboard(request, panel_id=None): + """Dashboard view to display insights.""" + + if not panel_id or panel_id not in PANELS: + return redirect("insights-panel", panel_id=PANELS[0]) + + panels = [] + for panel_key in PANELS: + panel_graphs = [graph for graph in GRAPHS.values() if graph.panel == panel_key] + panels.append( + { + "id": panel_key, + "label": PANEL_LABELS[panel_key], + "layout": PANEL_LAYOUTS.get(panel_key, "standard"), + "graphs": panel_graphs, + } + ) + + try: + snapshot = DailySnapshot.objects.latest() + last_time = snapshot.created_at + + snapshot_data = {} + for graph_id, graph_def in GRAPHS.items(): + if graph_def.data_fn: + snapshot_data[graph_id] = graph_def.data_fn(snapshot) + except DailySnapshot.DoesNotExist: + last_time = None + snapshot_data = {} + + search_query = request.GET.get("q", "").strip() + search_results_dict = None + search_error = None + + if search_query: + if panel_id == "severity_panel": + search_results_dict, search_error = _severity_search(search_query) + elif panel_id == "package_panel": + search_results_dict, search_error = _cwe_search(search_query) + + context = { + "panels": panels, + "current_panel_id": panel_id, + "last_snapshot_time": last_time, + "snapshot_dict": {"snapshot_data": snapshot_data}, + "search_query": search_query, + "search_results_dict": search_results_dict, + "search_error": search_error, + } + return render(request, "insights/dashboard.html", context) diff --git a/vulnerabilities/improvers/__init__.py b/vulnerabilities/improvers/__init__.py index 51694bde9..aeaa57654 100644 --- a/vulnerabilities/improvers/__init__.py +++ b/vulnerabilities/improvers/__init__.py @@ -7,6 +7,7 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +from insights.insights_snapshot_pipeline import InsightsSnapshotPipeline from vulnerabilities.pipelines.v2_improvers import archive_urls from vulnerabilities.pipelines.v2_improvers import collect_ssvc_trees from vulnerabilities.pipelines.v2_improvers import compute_advisory_todo as compute_advisory_todo_v2 @@ -43,5 +44,6 @@ enhance_with_github_poc.GithubPocsImproverPipeline, mark_unfurl_version_range.MarkUnfurlVersionRangePipeline, group_advisories_for_packages_v2.GroupAdvisoriesForPackages, + InsightsSnapshotPipeline, ] ) diff --git a/vulnerabilities/templates/navbar.html b/vulnerabilities/templates/navbar.html index 530c2521e..6e9979019 100644 --- a/vulnerabilities/templates/navbar.html +++ b/vulnerabilities/templates/navbar.html @@ -26,6 +26,9 @@
+ + Insights + Advisory To-Dos diff --git a/vulnerablecode/settings.py b/vulnerablecode/settings.py index a4c9705dc..d8fa6c7b1 100644 --- a/vulnerablecode/settings.py +++ b/vulnerablecode/settings.py @@ -78,6 +78,7 @@ # Local apps # Must come before Third-party apps for proper templates override "vulnerabilities", + "insights", # Django built-in "django.contrib.auth", "django.contrib.contenttypes", diff --git a/vulnerablecode/urls.py b/vulnerablecode/urls.py index b693bd725..125fdd620 100644 --- a/vulnerablecode/urls.py +++ b/vulnerablecode/urls.py @@ -65,6 +65,7 @@ def __init__(self, *args, **kwargs): urlpatterns = [ path("admin/login/", AdminLoginView.as_view(), name="admin-login"), path("api/v3/", include(api_v3_router.urls)), + path("insights/", include("insights.urls")), path( "robots.txt", TemplateView.as_view(template_name="robots.txt", content_type="text/plain"),