+
Skip to content

Enhance the Package search logic #160 #161

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Aug 7, 2024
Merged

Enhance the Package search logic #160 #161

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions component_catalog/filters.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,13 +159,22 @@ def filter(self, qs, value):
class PackageFilterSet(DataspacedFilterSet):
q = PackageSearchFilter(
label=_("Search"),
match_order_fields=["filename"],
search_fields=[
match_order_fields=[
"type",
"namespace",
"name",
"version",
"filename",
"download_url",
"sha1",
"md5",
],
search_fields=[
"type",
"namespace",
"name",
"version",
"filename",
"download_url",
"sha1",
"md5",
Expand Down
18 changes: 18 additions & 0 deletions component_catalog/tests/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,21 @@
# See https://github.com/nexB/dejacode for support or download.
# See https://aboutcode.org for more information about AboutCode FOSS projects.
#

from component_catalog.models import Component
from component_catalog.models import Package


def make_package(dataspace, package_url=None, **data):
package = Package(dataspace=dataspace, **data)
if package_url:
package.set_package_url(package_url)
package.save()
return package


def make_component(dataspace, **data):
return Component.objects.create(
dataspace=dataspace,
**data,
)
101 changes: 53 additions & 48 deletions component_catalog/tests/test_filters.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@
from component_catalog.models import Component
from component_catalog.models import ComponentKeyword
from component_catalog.models import ComponentType
from component_catalog.models import Package
from component_catalog.tests import make_component
from component_catalog.tests import make_package
from dje.models import Dataspace
from dje.tests import create_superuser
from dje.tests import create_user
Expand All @@ -28,7 +29,7 @@

class ComponentFilterSetTest(TestCase):
def setUp(self):
self.dataspace = Dataspace.objects.create(name="nexB")
self.dataspace = Dataspace.objects.create(name="Reference")
self.other_dataspace = Dataspace.objects.create(name="Other")
self.nexb_user = create_superuser("nexb_user", self.dataspace)
self.basic_user = create_user("basic_user", self.dataspace)
Expand Down Expand Up @@ -123,10 +124,8 @@ def test_component_filterset_primary_language_filter(self):
filterset = ComponentFilterSet(dataspace=self.dataspace)
self.assertEqual([], list(filterset.filters["primary_language"].field.choices))

c1 = Component.objects.create(
name="c1", dataspace=self.dataspace, primary_language="Python"
)
c2 = Component.objects.create(name="c2", dataspace=self.dataspace, primary_language="Java")
c1 = make_component(self.dataspace, name="c1", primary_language="Python")
c2 = make_component(self.dataspace, name="c2", primary_language="Java")

filterset = ComponentFilterSet(dataspace=self.dataspace)
self.assertEqual([c1, c2], list(filterset.qs))
Expand All @@ -146,13 +145,13 @@ def test_component_filterset_related_only_values_filter(self):
["licenses", "primary_language", "usage_policy"], ComponentFilterSet.related_only
)

c1 = Component.objects.create(
c1 = make_component(
self.dataspace,
name="c1",
dataspace=self.dataspace,
license_expression=self.license1.key,
primary_language="Python",
)
c2 = Component.objects.create(name="c2", dataspace=self.dataspace, primary_language="Java")
c2 = make_component(self.dataspace, name="c2", primary_language="Java")

filterset = ComponentFilterSet(dataspace=self.dataspace)
self.assertEqual([c1, c2], list(filterset.qs))
Expand Down Expand Up @@ -210,9 +209,9 @@ class ComponentFilterSearchTestCase(TestCase):
fixtures = [join(testfiles_location, "search", "component_dataset.json")]

def test_component_filterset_search_filter(self):
nexb_dataspace = Dataspace.objects.get(name="nexB")
dataspace = Dataspace.objects.get(name="Reference")
data = {"q": ""}
component_filterset = ComponentFilterSet(dataspace=nexb_dataspace, data=data)
component_filterset = ComponentFilterSet(dataspace=dataspace, data=data)
expected = [
"jblogbackup 1.0",
"jblogbackup 1.1",
Expand All @@ -227,7 +226,7 @@ def test_component_filterset_search_filter(self):
self.assertEqual(expected, [str(component) for component in component_filterset.qs])

data = {"q": "logback"}
component_filterset = ComponentFilterSet(dataspace=nexb_dataspace, data=data)
component_filterset = ComponentFilterSet(dataspace=dataspace, data=data)
expected = [
"logback 0.9.9",
"logback 1.0.0",
Expand All @@ -241,57 +240,57 @@ def test_component_filterset_search_filter(self):
self.assertEqual(expected, [str(component) for component in component_filterset.qs])

def test_component_filterset_sort_keeps_default_ordering_from_model(self):
nexb_dataspace = Dataspace.objects.get(name="nexB")
dataspace = Dataspace.objects.get(name="Reference")
data = {}
component_filterset = ComponentFilterSet(dataspace=nexb_dataspace, data=data)
component_filterset = ComponentFilterSet(dataspace=dataspace, data=data)
self.assertEqual((), component_filterset.qs.query.order_by)

data = {"sort": ""}
component_filterset = ComponentFilterSet(dataspace=nexb_dataspace, data=data)
component_filterset = ComponentFilterSet(dataspace=dataspace, data=data)
self.assertEqual((), component_filterset.qs.query.order_by)

data = {"sort": "invalid"}
component_filterset = ComponentFilterSet(dataspace=nexb_dataspace, data=data)
component_filterset = ComponentFilterSet(dataspace=dataspace, data=data)
self.assertEqual((), component_filterset.qs.query.order_by)

data = {"sort": "name"}
component_filterset = ComponentFilterSet(dataspace=nexb_dataspace, data=data)
component_filterset = ComponentFilterSet(dataspace=dataspace, data=data)
self.assertEqual(("name", "version"), component_filterset.qs.query.order_by)

data = {"sort": "version"}
component_filterset = ComponentFilterSet(dataspace=nexb_dataspace, data=data)
component_filterset = ComponentFilterSet(dataspace=dataspace, data=data)
self.assertEqual(("version", "name"), component_filterset.qs.query.order_by)

data = {"sort": "primary_language"}
component_filterset = ComponentFilterSet(dataspace=nexb_dataspace, data=data)
component_filterset = ComponentFilterSet(dataspace=dataspace, data=data)
self.assertEqual(
("primary_language", "name", "version"), component_filterset.qs.query.order_by
)


class PackageFilterSearchTestCase(TestCase):
def setUp(self):
self.nexb_dataspace = Dataspace.objects.create(name="nexB")
def sorted_results(self, qs):
return sorted([str(package) for package in qs])

def create_package(**kwargs):
return Package.objects.create(**kwargs, dataspace=self.nexb_dataspace)
def setUp(self):
self.dataspace = Dataspace.objects.create(name="Reference")

filename = {
"filename": "setup.exe",
}
create_package(**filename)
make_package(self.dataspace, **filename)

simple_purl = {
"type": "deb",
"name": "curl",
}
create_package(**simple_purl)
make_package(self.dataspace, **simple_purl)

simple_purl2 = {
**simple_purl,
"type": "git",
}
create_package(**simple_purl2)
make_package(self.dataspace, **simple_purl2)

complete_purl = {
"type": "deb",
Expand All @@ -301,60 +300,66 @@ def create_package(**kwargs):
"qualifiers": "arch=i386",
"subpath": "googleapis/api/annotations",
}
create_package(**complete_purl)
make_package(self.dataspace, **complete_purl)

def test_package_filterset_search_filter(self):
def sorted_results(qs):
return sorted([str(package) for package in qs])

data = {"q": ""}
filterset = PackageFilterSet(dataspace=self.nexb_dataspace, data=data)
filterset = PackageFilterSet(dataspace=self.dataspace, data=data)
expected = [
"pkg:deb/debian/curl@7.50.3-1?arch=i386#googleapis/api/annotations",
"pkg:git/curl",
"pkg:deb/curl",
"setup.exe",
]
self.assertEqual(sorted(expected), sorted_results(filterset.qs))
self.assertEqual(sorted(expected), self.sorted_results(filterset.qs))

data = {"q": "deb/curl"}
filterset = PackageFilterSet(dataspace=self.nexb_dataspace, data=data)
filterset = PackageFilterSet(dataspace=self.dataspace, data=data)
expected = [
"pkg:deb/curl",
"pkg:deb/debian/curl@7.50.3-1?arch=i386#googleapis/api/annotations",
]
self.assertEqual(sorted(expected), sorted_results(filterset.qs))
self.assertEqual(sorted(expected), self.sorted_results(filterset.qs))
data = {"q": "pkg:deb/curl"}
filterset = PackageFilterSet(dataspace=self.nexb_dataspace, data=data)
self.assertEqual(sorted(expected), sorted_results(filterset.qs))
filterset = PackageFilterSet(dataspace=self.dataspace, data=data)
self.assertEqual(sorted(expected), self.sorted_results(filterset.qs))

data = {"q": "deb/debian/curl@7.50.3-1"}
filterset = PackageFilterSet(dataspace=self.nexb_dataspace, data=data)
filterset = PackageFilterSet(dataspace=self.dataspace, data=data)
expected = [
"pkg:deb/debian/curl@7.50.3-1?arch=i386#googleapis/api/annotations",
]
self.assertEqual(sorted(expected), sorted_results(filterset.qs))
self.assertEqual(sorted(expected), self.sorted_results(filterset.qs))
data = {"q": "pkg:deb/debian/curl@7.50.3-1"}
filterset = PackageFilterSet(dataspace=self.nexb_dataspace, data=data)
self.assertEqual(sorted(expected), sorted_results(filterset.qs))
filterset = PackageFilterSet(dataspace=self.dataspace, data=data)
self.assertEqual(sorted(expected), self.sorted_results(filterset.qs))

data = {"q": "git/curl"}
filterset = PackageFilterSet(dataspace=self.nexb_dataspace, data=data)
filterset = PackageFilterSet(dataspace=self.dataspace, data=data)
expected = [
"pkg:git/curl",
]
self.assertEqual(sorted(expected), sorted_results(filterset.qs))
self.assertEqual(sorted(expected), self.sorted_results(filterset.qs))
data = {"q": "pkg:git/curl"}
filterset = PackageFilterSet(dataspace=self.nexb_dataspace, data=data)
self.assertEqual(sorted(expected), sorted_results(filterset.qs))
filterset = PackageFilterSet(dataspace=self.dataspace, data=data)
self.assertEqual(sorted(expected), self.sorted_results(filterset.qs))

data = {"q": "setup.exe"}
filterset = PackageFilterSet(dataspace=self.nexb_dataspace, data=data)
filterset = PackageFilterSet(dataspace=self.dataspace, data=data)
expected = [
"setup.exe",
]
self.assertEqual(sorted(expected), sorted_results(filterset.qs))
self.assertEqual(sorted(expected), self.sorted_results(filterset.qs))

data = {"q": "pkg:setup.exe"}
filterset = PackageFilterSet(dataspace=self.nexb_dataspace, data=data)
self.assertEqual([], sorted_results(filterset.qs))
filterset = PackageFilterSet(dataspace=self.dataspace, data=data)
self.assertEqual([], self.sorted_results(filterset.qs))

def test_package_filterset_search_match_order_on_purl_fields(self):
make_package(self.dataspace, package_url="pkg:pypi/django@5.0")
make_package(self.dataspace, package_url="pkg:pypi/django@4.0", filename="Django-4.0.zip")

data = {"q": "django"}
filterset = PackageFilterSet(dataspace=self.dataspace, data=data)
expected = ["pkg:pypi/django@4.0", "pkg:pypi/django@5.0"]
self.assertEqual(sorted(expected), self.sorted_results(filterset.qs))
20 changes: 10 additions & 10 deletions component_catalog/tests/testfiles/search/component_dataset.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"model": "dje.dataspace",
"fields": {
"uuid": "f26d21dc-d781-4727-84c3-1f7edc14b919",
"name": "nexB",
"name": "Reference",
"homepage_url": "http://www.nexb.com/",
"contact_info": "http://www.nexb.com/",
"notes": "Creators/maintainers of the system software",
Expand All @@ -25,7 +25,7 @@
"model": "component_catalog.component",
"fields": {
"dataspace": [
"nexB"
"Reference"
],
"uuid": "53fa7229-617b-4dfa-9eee-8c09e9380046",
"created_date": "2014-07-01T00:00:00Z",
Expand Down Expand Up @@ -65,7 +65,7 @@
"model": "component_catalog.component",
"fields": {
"dataspace": [
"nexB"
"Reference"
],
"uuid": "03df48a9-60df-43b6-9c04-4a4c9015f309",
"created_date": "2014-07-01T00:00:00Z",
Expand Down Expand Up @@ -105,7 +105,7 @@
"model": "component_catalog.component",
"fields": {
"dataspace": [
"nexB"
"Reference"
],
"uuid": "fed79fab-5d00-4994-a2d1-ad35164b736a",
"created_date": "2014-07-01T00:00:00Z",
Expand Down Expand Up @@ -145,7 +145,7 @@
"model": "component_catalog.component",
"fields": {
"dataspace": [
"nexB"
"Reference"
],
"uuid": "4a037d63-f479-41f7-985e-436b8039160a",
"created_date": "2014-07-01T00:00:00Z",
Expand Down Expand Up @@ -185,7 +185,7 @@
"model": "component_catalog.component",
"fields": {
"dataspace": [
"nexB"
"Reference"
],
"uuid": "663fe571-6042-48cd-95ec-6cf57316a591",
"created_date": "2015-08-04T15:08:45.738Z",
Expand Down Expand Up @@ -225,7 +225,7 @@
"model": "component_catalog.component",
"fields": {
"dataspace": [
"nexB"
"Reference"
],
"uuid": "d47238fd-09aa-45e9-aafa-9d8c3fe10b93",
"created_date": "2014-07-01T00:00:00Z",
Expand Down Expand Up @@ -265,7 +265,7 @@
"model": "component_catalog.component",
"fields": {
"dataspace": [
"nexB"
"Reference"
],
"uuid": "31368d10-b832-4c77-87d3-2d300989d30e",
"created_date": "2014-07-01T00:00:00Z",
Expand Down Expand Up @@ -305,7 +305,7 @@
"model": "component_catalog.component",
"fields": {
"dataspace": [
"nexB"
"Reference"
],
"uuid": "0a8bfdd1-7c48-4ac7-99f1-9e6c46e1bb81",
"created_date": "2014-07-01T00:00:00Z",
Expand Down Expand Up @@ -345,7 +345,7 @@
"model": "component_catalog.component",
"fields": {
"dataspace": [
"nexB"
"Reference"
],
"uuid": "9b1c732d-081b-44ae-a366-ac0182281422",
"created_date": "2017-06-13T10:05:03.727Z",
Expand Down
2 changes: 1 addition & 1 deletion dje/filters.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ def filter(self, qs, value):

class MatchOrderedSearchFilter(SearchRankFilter):
"""
Start with a case-insensitive containment search on the `name` field,
Start with a case-insensitive containment search on the `match_order_fields` fields,
ordering based on the match type using annotations.

If that simple search Return nothing, fallback to the SearchRankFilter
Expand Down
点击 这是indexloc提供的php浏览器服务，不要输入任何密码和下载