aboutcode-org · tdruez · Jun 13, 2024 · Jun 11, 2024 · Jun 11, 2024 · Jun 11, 2024
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -15,6 +15,16 @@ Release notes
   modal form.
   https://github.com/nexB/dejacode/issues/128
 
+- Add support for PURL(s) in the "Add Package" modal.
+  If the PURL type is supported by the packageurl_python library, a download URL
+  will be generated for creating the package and submitting a scan.
+  https://github.com/nexB/dejacode/issues/131
+
+- Leverage PurlDB during the "Add Package" process.
+  DejaCode will look up the PurlDB to retrieve and fetch all available data to
+  create the package.
+  https://github.com/nexB/dejacode/issues/131
+
 ### Version 5.1.0
 
 - Upgrade Python version to 3.12 and Django to 5.0.x

diff --git a/component_catalog/models.py b/component_catalog/models.py
@@ -50,6 +50,7 @@
 from dejacode_toolkit.download import DataCollectionException
 from dejacode_toolkit.download import collect_package_data
 from dejacode_toolkit.purldb import PurlDB
+from dejacode_toolkit.purldb import pick_purldb_entry
 from dje import urn
 from dje.copier import post_copy
 from dje.copier import post_update
@@ -65,6 +66,7 @@
 from dje.models import ParentChildRelationshipModel
 from dje.models import ReferenceNotesMixin
 from dje.tasks import logger as tasks_logger
+from dje.utils import is_purl_str
 from dje.utils import set_fields_from_object
 from dje.validators import generic_uri_validator
 from dje.validators import validate_url_segment
@@ -93,6 +95,11 @@
 ]
 
 
+class PackageAlreadyExistsWarning(Exception):
+    def __init__(self, message):
+        self.message = message
+
+
 def validate_filename(value):
     invalid_chars = ["/", "\\", ":"]
     if any(char in value for char in invalid_chars):
@@ -2291,6 +2298,68 @@ def where_used(self, user):
             f"Component {self.component_set.count()}\n"
         )
 
+    @classmethod
+    def create_from_url(cls, url, user):
+        """
+        Create a package from the given URL for the specified user.
+
+        This function processes the URL to create a package entry. It handles
+        both direct download URLs and Package URLs (purls), checking for
+        existing packages to avoid duplicates. If the package is not already
+        present, it collects necessary package data and creates a new package
+        entry.
+        """
+        url = url.strip()
+        if not url:
+            return
+
+        package_data = {}
+        scoped_packages_qs = cls.objects.scope(user.dataspace)
+
+        if is_purl_str(url):
+            download_url = purl2url.get_download_url(url)
+            package_url = PackageURL.from_string(url)
+            existing_packages = scoped_packages_qs.for_package_url(url, exact_match=True)
+        else:
+            download_url = url
+            package_url = url2purl.get_purl(url)
+            existing_packages = scoped_packages_qs.filter(download_url=url)
+
+        if existing_packages:
+            package_links = [package.get_absolute_link() for package in existing_packages]
+            raise PackageAlreadyExistsWarning(
+                f"{url} already exists in your Dataspace as {', '.join(package_links)}"
+            )
+
+        # Matching in PurlDB early to avoid more processing in case of a match.
+        purldb_data = None
+        if user.dataspace.enable_purldb_access:
+            package_for_match = cls(download_url=download_url)
+            package_for_match.set_package_url(package_url)
+            purldb_entries = package_for_match.get_purldb_entries(user)
+            # Look for one ith the same exact purl in that case
+            if purldb_data := pick_purldb_entry(purldb_entries, purl=url):
+                # The format from PurlDB is "2019-11-18T00:00:00Z" from DateTimeField
+                if release_date := purldb_data.get("release_date"):
+                    purldb_data["release_date"] = release_date.split("T")[0]
+                package_data.update(purldb_data)
+
+        if download_url and not purldb_data:
+            package_data = collect_package_data(download_url)
+
+        if sha1 := package_data.get("sha1"):
+            if sha1_match := scoped_packages_qs.filter(sha1=sha1):
+                package_link = sha1_match[0].get_absolute_link()
+                raise PackageAlreadyExistsWarning(
+                    f"{url} already exists in your Dataspace as {package_link}"
+                )
+
+        if package_url:
+            package_data.update(package_url.to_dict(encode=True, empty=""))
+
+        package = cls.create_from_data(user, package_data)
+        return package
+
     def get_purldb_entries(self, user, max_request_call=0, timeout=None):
         """
         Return the PurlDB entries that correspond to this Package instance.

diff --git a/component_catalog/templates/component_catalog/includes/add_package_modal.html b/component_catalog/templates/component_catalog/includes/add_package_modal.html
@@ -10,19 +10,28 @@ <h5 class="modal-title">Add Package</h5>
       <form id="package-add-form" method="post" data-add-url="{% url "component_catalog:package_add_urls" %}">{% csrf_token %}
         <div class="modal-body bg-body-tertiary">
           <div>
-            <label for="download-urls" class="form-label">Download URL(s)</label>
-            <textarea class="form-control" id="download-urls" aria-describedby="download-urls-help" placeholder="https://..."></textarea>
+            <label for="download-urls" class="form-label">Download URL(s) and/or Package URL(s)</label>
+            <textarea class="form-control" id="download-urls" aria-describedby="download-urls-help" placeholder="https://&#10;pkg:"></textarea>
             <small id="download-urls-help" class="form-text text-muted">
-              Enter the download URL for obtaining the package.<br>
+              Enter the download URL(s) or Package URL(s) of your packages.<br><br>
               You can provide multiple URLs separated by a new-line.<br>
               A download URL is one that will immediately initiate the download of a software package if you click it on a browser page or paste it into a browser address field.<br><br>
               Note that this feature is intended only for <strong>publicly available open source packages</strong>, not your private code.<br>
               DejaCode will automatically collect the <code>filename</code>, <code>sha1</code>, <code>md5</code>, and <code>size</code> and apply them to the package definition.
             </small>
             {% if user.dataspace.enable_package_scanning %}
-              <small class="form-text text-muted mt-3">
-                <strong>Package scanning is enabled in your Dataspace</strong>, DejaCode will also submit the package to ScanCode.io and the results will be returned to the "Scan" detail tab of the package when that scan is complete.
-              </small>
+              <div class="mt-3">
+                <small class="form-text text-muted mt-3">
+                  <strong>Package scanning is enabled in your Dataspace</strong>, DejaCode will also submit the package to ScanCode.io and the results will be returned to the "Scan" detail tab of the package when that scan is complete.
+                </small>
+              </div>
+            {% endif %}
+            {% if user.dataspace.enable_purldb_access %}
+              <div class="mt-3">
+                <small class="form-text text-muted mt-3">
+                  <strong>PurlDB access is enabled in your Dataspace</strong>, DejaCode will look up the PurlDB to retrieve and fetch all available data to create the package.
+                </small>
+              </div>
             {% endif %}
             <div id="package-add-error" class="alert alert-danger mt-3" role="alert" style="display: none;"></div>
           </div>
@@ -66,8 +75,13 @@ <h5 class="modal-title">Add Package</h5>
                 },
                 error: function(data) {
                     $('#overlay').remove();
-                    let error_msg = data;
-                    if (data.responseJSON) error_msg = data.responseJSON.error_message;
+                    let error_msg;
+                    if (data.responseJSON)  {
+                        error_msg = data.responseJSON.error_message;
+                    }
+                    else {
+                        error_msg = data.statusText;
+                    }
                     $('#package-add-error').html('<strong>Error:</strong> ' + error_msg).show();
                 }
             });

diff --git a/component_catalog/tests/test_models.py b/component_catalog/tests/test_models.py
@@ -35,6 +35,7 @@
 from component_catalog.models import ComponentType
 from component_catalog.models import LicenseExpressionMixin
 from component_catalog.models import Package
+from component_catalog.models import PackageAlreadyExistsWarning
 from component_catalog.models import Subcomponent
 from dejacode_toolkit import download
 from dejacode_toolkit.download import DataCollectionException
@@ -1644,6 +1645,70 @@ def test_package_model_update_from_data(self):
         package.refresh_from_db()
         self.assertEqual(new_data["name"], package.name)
 
+    @mock.patch("component_catalog.models.collect_package_data")
+    def test_package_model_create_from_url(self, mock_collect):
+        self.assertIsNone(Package.create_from_url(url=" ", user=self.user))
+
+        download_url = "https://dejacode.com/archive.zip"
+        mock_collect.return_value = {
+            "download_url": download_url,
+            "filename": "archive.zip",
+        }
+        package = Package.create_from_url(url=download_url, user=self.user)
+        self.assertTrue(package.uuid)
+        self.assertEqual(self.user, package.created_by)
+        expected = "pkg:generic/archive.zip?download_url=https://dejacode.com/archive.zip"
+        self.assertEqual(expected, package.package_url)
+        self.assertEqual(download_url, package.download_url)
+
+        with self.assertRaises(PackageAlreadyExistsWarning) as cm:
+            Package.create_from_url(url=download_url, user=self.user)
+        self.assertIn("already exists in your Dataspace", cm.exception.message)
+
+        purl = "pkg:npm/is-npm@1.0.0"
+        mock_collect.return_value = {}
+        package = Package.create_from_url(url=purl, user=self.user)
+        self.assertTrue(package.uuid)
+        self.assertEqual(self.user, package.created_by)
+        self.assertEqual(purl, package.package_url)
+        mock_collect.assert_called_with("http://registry.npmjs.org/is-npm/-/is-npm-1.0.0.tgz")
+
+    @mock.patch("component_catalog.models.Package.get_purldb_entries")
+    @mock.patch("dejacode_toolkit.purldb.PurlDB.is_configured")
+    def test_package_model_create_from_url_enable_purldb_access(
+        self, mock_is_configured, mock_get_purldb_entries
+    ):
+        self.dataspace.enable_purldb_access = True
+        self.dataspace.save()
+        mock_is_configured.return_value = True
+        purldb_entry = {
+            "uuid": "7b947095-ab4c-45e3-8af3-6a73bd88e31d",
+            "filename": "abbot-1.4.0.jar",
+            "release_date": "2023-02-01T00:27:00Z",
+            "type": "maven",
+            "namespace": "abbot",
+            "name": "abbot",
+            "version": "1.4.0",
+            "primary_language": "Java",
+            "description": "Abbot Java GUI Test Library",
+            "keywords": ["keyword1", "keyword2"],
+            "homepage_url": "http://abbot.sf.net/",
+            "download_url": "http://repo1.maven.org/maven2/abbot/abbot/" "1.4.0/abbot-1.4.0.jar",
+            "size": 687192,
+            "sha1": "a2363646a9dd05955633b450010b59a21af8a423",
+            "license_expression": "(bsd-new OR eps-1.0 OR apache-2.0 OR mit) AND unknown",
+            "declared_license": "EPL\nhttps://www.eclipse.org/legal/eps-v10.html",
+            "package_url": "pkg:maven/abbot/abbot@1.4.0",
+        }
+        mock_get_purldb_entries.return_value = [purldb_entry]
+
+        purl = "pkg:maven/abbot/abbot@1.4.0"
+        package = Package.create_from_url(url=purl, user=self.user)
+        mock_get_purldb_entries.assert_called_once()
+        self.assertEqual(self.user, package.created_by)
+        for field_name, value in purldb_entry.items():
+            self.assertEqual(value, getattr(package, field_name))
+
     def test_package_model_get_url_methods(self):
         package = Package(
             filename="filename.zip",

diff --git a/component_catalog/tests/test_views.py b/component_catalog/tests/test_views.py
@@ -1632,12 +1632,15 @@ def test_package_create_ajax_view(self):
 
         self.client.login(username=self.basic_user.username, password="secret")
         response = self.client.get(package_add_url)
+        self.assertEqual(405, response.status_code)
+
+        response = self.client.post(package_add_url)
         self.assertEqual(403, response.status_code)
         expected = {"error_message": "Permission denied"}
         self.assertEqual(expected, response.json())
 
         self.client.login(username=self.super_user.username, password="secret")
-        response = self.client.get(package_add_url)
+        response = self.client.post(package_add_url)
         self.assertEqual(400, response.status_code)
         expected = {"error_message": "Missing Download URL"}
         self.assertEqual(expected, response.json())
@@ -1664,7 +1667,7 @@ def test_package_create_ajax_view(self):
         response = self.client.get("/packages/")
         messages = list(response.context["messages"])
         msg = (
-            f"URL https://dejacode.com/archive.zip already exists in your Dataspace as "
+            f"https://dejacode.com/archive.zip already exists in your Dataspace as "
             f'<a href="{self.package1.get_absolute_url()}">package1</a>'
         )
         self.assertEqual(str(messages[0]), msg)
@@ -1678,7 +1681,7 @@ def test_package_create_ajax_view(self):
             "sha1": "5ba93c9db0cff93f52b521d7420e43f6eda2784f",
             "md5": "93b885adfe0da089cdf634904fd59f71",
         }
-        with mock.patch("component_catalog.views.collect_package_data") as collect:
+        with mock.patch("component_catalog.models.collect_package_data") as collect:
             collect.return_value = collected_data
             response = self.client.post(package_add_url, data)
 
@@ -1701,16 +1704,16 @@ def test_package_create_ajax_view(self):
         # Different URL but sha1 match in the db
         data = {"download_urls": "https://url.com/file.ext"}
         collected_data["download_url"] = data["download_urls"]
-        with mock.patch("component_catalog.views.collect_package_data") as collect:
+        with mock.patch("component_catalog.models.collect_package_data") as collect:
             collect.return_value = collected_data
             response = self.client.post(package_add_url, data)
 
         self.assertEqual(200, response.status_code)
         response = self.client.get("/packages/")
         messages = list(response.context["messages"])
         msg = (
-            f'The package at URL {collected_data["download_url"]} already exists in'
-            f' your Dataspace as <a href="{new_package.get_absolute_url()}">{new_package}</a>'
+            f'{collected_data["download_url"]} already exists in your Dataspace as '
+            f'<a href="{new_package.get_absolute_url()}">{new_package}</a>'
         )
         self.assertEqual(str(messages[0]), msg)
         self.assertFalse(