Merge branch 'dev' into merge_mobsf

Author: manuel-sommer

.github/workflows/release-x-manual-helm-chart.yml

@@ -22,6 +22,21 @@ on:
         description: 'Release number'
         required: true
 
+      make_draft:
+        type: boolean
+        description: 'Mark as draft release?'
+        default: true
+
+      make_prerelease:
+        type: boolean
+        description: 'Mark as pre-release?'
+        default: false
+
+      make_latest:
+        type: boolean
+        description: 'Mark as latest?'
+        default: false
+
 jobs:
   release-chart:
     runs-on: ubuntu-latest
@@ -38,7 +53,7 @@ jobs:
       #   id: get-upload-url
       #   uses: pdamianik/[email protected]
       #   with:
-      #     tag: ${{ github.event.inputs.release_number }}
+      #     tag: ${{ inputs.release_number }}
       #     token: ${{ github.token }}
 
       - name: Configure git
@@ -62,24 +77,25 @@ jobs:
         id: pin_image
         run: |-
           yq --version
-          yq -i '.tag="${{ github.event.inputs.release_number }}"'  helm/defectdojo/values.yaml
+          yq -i '.tag="${{ inputs.release_number }}"'  helm/defectdojo/values.yaml
           echo "Current image tag:`yq -r '.tag' helm/defectdojo/values.yaml`"
 
       - name: Package Helm chart
         id: package-helm-chart
         run: |
           mkdir build
           helm package helm/defectdojo/ --destination ./build
-          echo "chart_version=$(ls build | cut -d '-' -f 2 | sed 's|\.tgz||')" >> $GITHUB_ENV
+          echo "chart_version=$(ls build | cut -d '-' -f 2,3 | sed 's|\.tgz||')" >> $GITHUB_ENV
 
-      - name: Create release ${{ github.event.inputs.release_number }}
+      - name: Create release ${{ inputs.release_number }}
         uses: softprops/action-gh-release@da05d552573ad5aba039eaac05058a918a7bf631 # v2.2.2
         with:
-          name: '${{ github.event.inputs.release_number }} 🌈'
-          tag_name: ${{ github.event.inputs.release_number }}
+          name: '${{ inputs.release_number }} 🌈'
+          tag_name: ${{ inputs.release_number }}
           body: Run the release drafter to populate the release notes.
-          draft: true
-          prerelease: false
+          draft: ${{ inputs.make_draft }}
+          prerelease: ${{ inputs.make_prerelease }}
+          make_latest: ${{ inputs.make_latest }}
           files: ./build/defectdojo-${{ env.chart_version }}.tgz
           token: ${{ secrets.GITHUB_TOKEN }}
         env:
@@ -96,9 +112,9 @@ jobs:
           git checkout helm-charts
           git pull
           if [ ! -f ./index.yaml ]; then
-            helm repo index ./build --url "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/releases/download/${{ github.event.inputs.release_number }}/"
+            helm repo index ./build --url "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/releases/download/${{ inputs.release_number }}/"
           else
-            helm repo index ./build --url "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/releases/download/${{ github.event.inputs.release_number }}/" --merge ./index.yaml
+            helm repo index ./build --url "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/releases/download/${{ inputs.release_number }}/" --merge ./index.yaml
           fi
           cp -f ./build/index.yaml ./index.yaml
           git add ./index.yaml

.github/workflows/release-x-nightly.yml

@@ -77,5 +77,6 @@ jobs:
     uses: ./.github/workflows/release-x-manual-helm-chart.yml
     with:
       release_number: ${{ inputs.tag-to-apply }}
+      make_draft: false
+      make_prerelease: true
     secrets: inherit
-

components/package.json

@@ -1,6 +1,6 @@
 {
   "name": "defectdojo",
-  "version": "2.47.0-dev",
+  "version": "2.48.0-dev",
   "license" : "BSD-3-Clause",
   "private": true,
   "dependencies": {

docs/content/en/open_source/installation/running-in-production.md

@@ -88,21 +88,5 @@ You can execute the following command to see the configuration:
 `docker compose exec celerybeat bash -c "celery -A dojo inspect stats"`
 and see what is in effect.
 
-### Asynchronous Import
-
-<span style="background-color:rgba(242, 86, 29, 0.3)">This experimental feature has been deprecated as of DefectDojo 2.44.0 (March release).  Please exercise caution if using this feature with an older version of DefectDojo, as results may be inconsistent.</span>
-
-Import and Re-Import can also be configured to handle uploads asynchronously to aid in
-processing especially large scans. It works by batching Findings and Endpoints by a
-configurable amount. Each batch will be be processed in separate celery tasks.
-
-The following variables impact async imports.
-
--   `DD_ASYNC_FINDING_IMPORT` defaults to False
--   `DD_ASYNC_FINDING_IMPORT_CHUNK_SIZE` defaults to 100
-
-When using asynchronous imports with dynamic scanners, Endpoints will continue to "trickle" in
-even after the import has returned a successful response. This is because processing continues
-to occur after the Findings have already been imported.
-
-To determine if an import has been fully completed, please see the progress bar in the appropriate test.
+### Asynchronous Import: Deprecated
+This feature has been removed in 2.47.0

docs/content/en/open_source/upgrading/2.47.md

@@ -9,3 +9,7 @@ description: Drop support for PostgreSQL-HA in HELM
 This release removes support for the PostgreSQL-HA (High Availability) Helm chart as a dependency in the DefectDojo Helm chart. Users relying on the PostgreSQL-HA Helm chart will need to transition to using the standard PostgreSQL configuration or an external PostgreSQL database.
 
 There are no special instructions for upgrading to 2.47.x. Check the [Release Notes](https://github.com/DefectDojo/django-DefectDojo/releases/tag/2.47.0) for the contents of the release.
+
+## Removal of Asynchronous Import
+
+Please note that asynchronous import has been removed as it was announced in 2.46. If you haven't migrated from this feature yet, we recommend doing before upgrading to 2.47.0

docs/content/en/open_source/upgrading/2.48.md

@@ -1,9 +1,9 @@
 ---
 title: 'Upgrading to DefectDojo Version 2.48.x'
 toc_hide: true
-weight: -20250505
+weight: -20250602
 description: Recalculate hashes for MobSF parser
 ---
 ### Merging Mobsfscan Scan and MobSF Scan
 
-The two scan types Mobsfscan Scan and MobSF Scan were merged in this release. We recommend to recalculate the hashcodes if you use these parsers as the deduplication settings have been changed.
+The two scan types Mobsfscan Scan and MobSF Scan were merged in this release. We recommend to recalculate the hashcodes if you use these parsers as the deduplication settings have been changed.

dojo/__init__.py

@@ -4,6 +4,6 @@
 # Django starts so that shared_task will use this app.
 from .celery import app as celery_app  # noqa: F401
 
-__version__ = "2.47.0-dev"
+__version__ = "2.48.0-dev"
 __url__ = "https://github.com/DefectDojo/django-DefectDojo"
 __docs__ = "https://documentation.defectdojo.com"

dojo/api_v2/exception_handler.py

@@ -12,6 +12,7 @@
 from rest_framework.views import exception_handler
 
 from dojo.models import System_Settings
+from dojo.product_announcements import ErrorPageProductAnnouncement
 
 logger = logging.getLogger(__name__)
 
@@ -36,6 +37,7 @@ def custom_exception_handler(exc, context):
         response.status_code = HTTP_400_BAD_REQUEST
         response.data = {}
         response.data["message"] = str(exc)
+        ErrorPageProductAnnouncement(response=response)
     elif response is None:
         if System_Settings.objects.get().api_expose_error_details:
             exception_message = str(exc.args[0])
@@ -51,6 +53,7 @@ def custom_exception_handler(exc, context):
         response.data[
             "message"
         ] = exception_message
+        ErrorPageProductAnnouncement(response=response)
     elif response.status_code < 500:
         # HTTP status codes lower than 500 are no technical errors.
         # They need not to be logged and we provide the exception
@@ -60,6 +63,7 @@ def custom_exception_handler(exc, context):
             exc,
         ) != response.data.get("detail", ""):
             response.data["message"] = str(exc)
+            ErrorPageProductAnnouncement(response=response)
     else:
         # HTTP status code 500 or higher are technical errors.
         # They get logged and we don't change the response.

dojo/api_v2/serializers.py

@@ -3,6 +3,7 @@
 import json
 import logging
 import re
+import time
 from datetime import datetime
 
 import six
@@ -112,6 +113,10 @@
     Vulnerability_Id_Template,
     get_current_date,
 )
+from dojo.product_announcements import (
+    LargeScanSizeProductAnnouncement,
+    ScanTypeProductAnnouncement,
+)
 from dojo.tools.factory import (
     get_choices_sorted,
     requires_file,
@@ -2193,6 +2198,7 @@ class CommonImportScanSerializer(serializers.Serializer):
     product_id = serializers.IntegerField(read_only=True)
     product_type_id = serializers.IntegerField(read_only=True)
     statistics = ImportStatisticsSerializer(read_only=True, required=False)
+    pro = serializers.ListField(read_only=True, required=False)
     apply_tags_to_findings = serializers.BooleanField(
         help_text="If set to True, the tags will be applied to the findings",
         required=False,
@@ -2224,6 +2230,7 @@ def process_scan(
         Raises exceptions in the event of an error
         """
         try:
+            start_time = time.perf_counter()
             importer = self.get_importer(**context)
             context["test"], _, _, _, _, _, _ = importer.process_scan(
                 context.pop("scan", None),
@@ -2236,6 +2243,9 @@ def process_scan(
                 data["product_id"] = test.engagement.product.id
                 data["product_type_id"] = test.engagement.product.prod_type.id
                 data["statistics"] = {"after": test.statistics}
+            duration = time.perf_counter() - start_time
+            LargeScanSizeProductAnnouncement(response_data=data, duration=duration)
+            ScanTypeProductAnnouncement(response_data=data, scan_type=context.get("scan_type"))
         # convert to exception otherwise django rest framework will swallow them as 400 error
         # exceptions are already logged in the importer
         except SyntaxError as se:
@@ -2491,6 +2501,7 @@ def process_scan(
         """
         statistics_before, statistics_delta = None, None
         try:
+            start_time = time.perf_counter()
             if test := context.get("test"):
                 statistics_before = test.statistics
                 context["test"], _, _, _, _, _, test_import = self.get_reimporter(
@@ -2525,6 +2536,9 @@ def process_scan(
                 if statistics_delta:
                     data["statistics"]["delta"] = statistics_delta
                 data["statistics"]["after"] = test.statistics
+            duration = time.perf_counter() - start_time
+            LargeScanSizeProductAnnouncement(response_data=data, duration=duration)
+            ScanTypeProductAnnouncement(response_data=data, scan_type=context.get("scan_type"))
         # convert to exception otherwise django rest framework will swallow them as 400 error
         # exceptions are already logged in the importer
         except SyntaxError as se:

dojo/api_v2/views.py

@@ -52,6 +52,7 @@
 from dojo.filters import (
     ApiAppAnalysisFilter,
     ApiCredentialsFilter,
+    ApiDojoMetaFilter,
     ApiEndpointFilter,
     ApiEngagementFilter,
     ApiFindingFilter,
@@ -1643,14 +1644,7 @@ class DojoMetaViewSet(
     serializer_class = serializers.MetaSerializer
     queryset = DojoMeta.objects.none()
     filter_backends = (DjangoFilterBackend,)
-    filterset_fields = [
-        "id",
-        "product",
-        "endpoint",
-        "finding",
-        "name",
-        "value",
-    ]
+    filterset_class = ApiDojoMetaFilter
     permission_classes = (
         IsAuthenticated,
         permissions.UserHasDojoMetaPermission,