From 4c4b8363588832400f845857bfc83c7ef6414f0c Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Thu, 9 Jan 2025 18:27:24 +0100 Subject: [PATCH 01/24] Update for 3.8 (#1559) --- doc/what-is-new.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/what-is-new.md b/doc/what-is-new.md index d3ea79760..299f77581 100644 --- a/doc/what-is-new.md +++ b/doc/what-is-new.md @@ -1,3 +1,11 @@ +# Version 3.8 + +* Bug fixes +* Better compatibility with SonarQube 9.9, SonarCloud +* Better compatibility with SonarQube Community Build and Developer Edition +* Adjustments for SonarQube 10.8 (AI related features) +* Partial migration to users and groups APIv2 + # Version 3.7 * Numerous hardening, functional and performance improvements on `sonar-audit`, `sonar-measures-export`, `sonar-findings-export` From 01d482df9be9b3fbab079d8d2be9d1e9686d304c Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Thu, 9 Jan 2025 18:41:18 +0100 Subject: [PATCH 02/24] Bump Alpine version (#1562) --- conf/release.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/release.Dockerfile b/conf/release.Dockerfile index 95e35d609..bd7d1f5c6 100644 --- a/conf/release.Dockerfile +++ b/conf/release.Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.20.3 +FROM alpine:3.21.2 LABEL maintainer="olivier.korach@gmail.com" ENV IN_DOCKER="Yes" From 30b728f750088fc8a04001422fc59e2526406ee4 Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Thu, 9 Jan 2025 18:53:54 +0100 Subject: [PATCH 03/24] Bump version to 3.9 after release of 3.8 (#1563) * Bump version to 3.9 after release of 3.8 * Fix typo --- conf/release.Dockerfile | 2 +- doc/what-is-new.md | 2 ++ sonar/version.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/conf/release.Dockerfile b/conf/release.Dockerfile index bd7d1f5c6..be25c6764 100644 --- a/conf/release.Dockerfile +++ b/conf/release.Dockerfile @@ -31,7 +31,7 @@ COPY ./LICENSE . COPY ./sonar/audit sonar/audit RUN pip install --upgrade pip \ -&& pip install sonar-tools==3.8 +&& pip install sonar-tools==3.9 USER ${USERNAME} WORKDIR /home/${USERNAME} diff --git a/doc/what-is-new.md b/doc/what-is-new.md index 299f77581..fdb8571a4 100644 --- a/doc/what-is-new.md +++ b/doc/what-is-new.md @@ -1,3 +1,5 @@ +# Version 3.9 + # Version 3.8 * Bug fixes diff --git a/sonar/version.py b/sonar/version.py index 22258a3e2..12a234d8a 100644 --- a/sonar/version.py +++ b/sonar/version.py @@ -24,5 +24,5 @@ """ -PACKAGE_VERSION = "3.8" +PACKAGE_VERSION = "3.9" MIGRATION_TOOL_VERSION = "0.5" From d28eb22ce00943f4be7b813a6f4ba82c5381afc7 Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Mon, 20 Jan 2025 17:40:31 +0100 Subject: [PATCH 04/24] Improve Scan (#1568) * Add conf directory * Linters for external issues produce external* files * Improve issue headline * Better evaluation of local build * Fix alpine version * Quality pass * Quality pass --- conf/run_linters.sh | 14 +++++++------- conf/run_tests.sh | 4 ++-- conf/scan.sh | 33 +++++++++++++++++++++------------ conf/snapshot.Dockerfile | 2 +- conf/trivy2sonar.py | 8 ++++---- sonar-project.properties | 2 +- 6 files changed, 36 insertions(+), 27 deletions(-) diff --git a/conf/run_linters.sh b/conf/run_linters.sh index 0bbf825f4..f703dccf6 100755 --- a/conf/run_linters.sh +++ b/conf/run_linters.sh @@ -29,12 +29,12 @@ buildDir="$ROOTDIR/build" pylintReport="$buildDir/pylint-report.out" # banditReport="$buildDir/bandit-report.json" flake8Report="$buildDir/flake8-report.out" -shellcheckReport="$buildDir/shellcheck.json" -trivyReport="$buildDir/trivy.json" +shellcheckReport="$buildDir/external-issues-shellcheck.json" +trivyReport="$buildDir/external-issues-trivy.json" [ ! -d "$buildDir" ] && mkdir "$buildDir" # rm -rf -- ${buildDir:?"."}/* .coverage */__pycache__ */*.pyc # mediatools/__pycache__ tests/__pycache__ -echo "Running pylint" +echo "===> Running pylint" rm -f "$pylintReport" pylint --rcfile "$CONFDIR"/pylintrc "$ROOTDIR"/*.py "$ROOTDIR"/*/*.py -r n --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" | tee "$pylintReport" re=$? @@ -43,19 +43,19 @@ if [ "$re" == "32" ]; then exit $re fi -echo "Running flake8" +echo "===> Running flake8" rm -f "$flake8Report" # See .flake8 file for settings flake8 --config "$CONFIG/.flake8" "$ROOTDIR" >"$flake8Report" if [ "$localbuild" = "true" ]; then - echo "Running shellcheck" + echo "===> Running shellcheck" shellcheck "$ROOTDIR"/*.sh "$ROOTDIR"/*/*.sh -s bash -f json | "$CONFDIR"/shellcheck2sonar.py >"$shellcheckReport" - echo "Running checkov" + echo "===> Running checkov" checkov -d . --framework dockerfile -o sarif --output-file-path "$buildDir" - echo "Running trivy" + echo "===> Running trivy" "$CONFDIR"/build.sh docker trivy image -f json -o "$buildDir"/trivy_results.json olivierkorach/sonar-tools:latest python3 "$CONFDIR"/trivy2sonar.py < "$buildDir"/trivy_results.json > "$trivyReport" diff --git a/conf/run_tests.sh b/conf/run_tests.sh index 8d4a0d73d..277a41d34 100755 --- a/conf/run_tests.sh +++ b/conf/run_tests.sh @@ -19,12 +19,12 @@ # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # -ME="$( basename "${BASH_SOURCE[0]}" )" +# ME="$( basename "${BASH_SOURCE[0]}" )" ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )" CONFDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" buildDir="$ROOTDIR/build" -[ ! -d $buildDir ] && mkdir $buildDir +[ ! -d "$buildDir" ] && mkdir "$buildDir" echo "Running tests" diff --git a/conf/scan.sh b/conf/scan.sh index d49f68275..c107771d1 100755 --- a/conf/scan.sh +++ b/conf/scan.sh @@ -19,13 +19,17 @@ # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # -ME="$( basename "${BASH_SOURCE[0]}" )" +# ME="$( basename "${BASH_SOURCE[0]}" )" ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )" CONFDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" dolint="true" dotest="false" -localbuild="false" +if [ "$CI" == "" ]; then + localbuild="true" +else + localbuild="false" +fi scanOpts=() @@ -37,7 +41,6 @@ do ;; -test) dotest="true" - localbuild="true" ;; *) scanOpts=("${scanOpts[@]}" "$1") @@ -48,14 +51,9 @@ done buildDir="build" pylintReport="$buildDir/pylint-report.out" -banditReport="$buildDir/bandit-report.json" flake8Report="$buildDir/flake8-report.out" -coverageReport="$buildDir/coverage.xml" -shellcheckReport="$buildDir/shellcheck.json" -trivyReport="$buildDir/trivy.json" -utReport="$buildDir/xunit-results.xml" -[ ! -d $buildDir ] && mkdir $buildDir +[ ! -d "$buildDir" ] && mkdir "$buildDir" rm -rf -- ${buildDir:?"."}/* .coverage */__pycache__ */*.pyc # mediatools/__pycache__ testpytest/__pycache__ testunittest/__pycache__ @@ -67,12 +65,11 @@ if [ "$dotest" == "true" ]; then "$CONFDIR"/run_tests.sh fi -version=$(grep PACKAGE_VERSION $ROOTDIR/sonar/version.py | cut -d "=" -f 2 | sed -e "s/[\'\" ]//g" -e "s/^ +//" -e "s/ +$//") +version=$(grep PACKAGE_VERSION "$ROOTDIR/sonar/version.py" | cut -d "=" -f 2 | sed -e "s/[\'\" ]//g" -e "s/^ +//" -e "s/ +$//") cmd="sonar-scanner -Dsonar.projectVersion=$version \ -Dsonar.python.flake8.reportPaths=$flake8Report \ -Dsonar.python.pylint.reportPaths=$pylintReport \ - -Dsonar.externalIssuesReportPaths=$shellcheckReport,$trivyReport \ -Dsonar.login=$SONAR_TOKEN \ -Dsonar.token=$SONAR_TOKEN \ "${scanOpts[*]}"" @@ -82,6 +79,7 @@ if ls $buildDir/coverage*.xml >/dev/null 2>&1; then else echo "===> NO COVERAGE REPORT" fi + if ls $buildDir/xunit-results*.xml >/dev/null 2>&1; then cmd="$cmd -Dsonar.python.xunit.reportPath=$buildDir/xunit-results*.xml" else @@ -89,7 +87,18 @@ else cmd="$cmd -Dsonar.python.xunit.reportPath=" fi -echo "Running: $cmd" +if ls $buildDir/external-issues*.json >/dev/null 2>&1; then + files=$(ls $buildDir/external-issues*.json | tr '\n' ' ' | sed -E -e 's/ +$//' -e 's/ +/,/g') + echo "EXTERNAL ISSUES FILES = $files" + cmd="$cmd -Dsonar.externalIssuesReportPaths=$files" +else + echo "===> NO EXTERNAL ISSUES" +fi + + +echo +echo "Running: $cmd" | sed "s/$SONAR_TOKEN//g" +echo $cmd diff --git a/conf/snapshot.Dockerfile b/conf/snapshot.Dockerfile index 9fe3dc99f..5b398eb3e 100644 --- a/conf/snapshot.Dockerfile +++ b/conf/snapshot.Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.20.3 +FROM alpine:3.21.2 LABEL maintainer="olivier.korach@gmail.com" ENV IN_DOCKER="Yes" diff --git a/conf/trivy2sonar.py b/conf/trivy2sonar.py index 1e62c64d0..50606789c 100755 --- a/conf/trivy2sonar.py +++ b/conf/trivy2sonar.py @@ -36,7 +36,7 @@ def main() -> None: text = "".join(sys.stdin) rules_dict = {} - issue_list = [] + issue_list = {} for issue in json.loads(text)["Results"][0]["Vulnerabilities"]: @@ -44,7 +44,7 @@ def main() -> None: "ruleId": f"{TOOLNAME}:{issue['VulnerabilityID']}", "effortMinutes": 30, "primaryLocation": { - "message": issue["Title"], + "message": f"{issue['VulnerabilityID']} - {issue['Title']}", "filePath": "conf/snapshot.Dockerfile", "textRange": { "startLine": 1, @@ -54,7 +54,7 @@ def main() -> None: }, }, } - issue_list.append(sonar_issue) + issue_list[sonar_issue["primaryLocation"]["message"]] = sonar_issue # score = max([v["V3Score"] for v in issue['CVSS'].values()]) # if score <= 4: # sev = "LOW" @@ -74,7 +74,7 @@ def main() -> None: "impacts": [{"softwareQuality": "SECURITY", "severity": sev_mqr}], } - external_issues = {"rules": list(rules_dict.values()), "issues": issue_list} + external_issues = {"rules": list(rules_dict.values()), "issues": list(issue_list.values())} print(json.dumps(external_issues, indent=3, separators=(",", ": "))) diff --git a/sonar-project.properties b/sonar-project.properties index a21531f60..f608a347c 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -4,7 +4,7 @@ sonar.projectName=Sonar Tools sonar.python.version=3.9 # Comma-separated paths to directories with sources (required) -sonar.sources=sonar, cli, migration, setup.py, setup_migration.py +sonar.sources=sonar, cli, migration, conf, setup.py, setup_migration.py # Encoding of the source files sonar.sourceEncoding=UTF-8 From 6ec7c4e4d8174869f17c8de097e03aae58f67b0b Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Wed, 12 Feb 2025 17:28:37 +0100 Subject: [PATCH 05/24] Change IT tools color scheme (#1571) --- test/integration/it-tools.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/it-tools.sh b/test/integration/it-tools.sh index 93c085744..83f78e5d7 100644 --- a/test/integration/it-tools.sh +++ b/test/integration/it-tools.sh @@ -26,7 +26,7 @@ TMP="$REPO_ROOT/tmp" IT_LOG_FILE="$TMP/it.log" mkdir -p "$TMP" -RED=$(tput setaf 1) +YELLOW=$(tput setaf 3) GREEN=$(tput setaf 2) RESET=$(tput setaf 7) From 832a8de3955f508312906cc136a23ac10a676fb9 Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Wed, 12 Feb 2025 18:45:28 +0100 Subject: [PATCH 06/24] Fix sonar-projects import check is too strict (#1572) --- cli/projects_cli.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cli/projects_cli.py b/cli/projects_cli.py index 5141edba1..243541487 100644 --- a/cli/projects_cli.py +++ b/cli/projects_cli.py @@ -59,9 +59,10 @@ def __check_sq_environments(import_sq: platform.Platform, export_sq: dict[str, s raise exceptions.UnsupportedOperation( f"Export was not performed with same SonarQube version, aborting... ({utilities.version_to_string(exp_version)} vs {utilities.version_to_string(imp_version)})" ) - if export_sq["plugins"] != import_sq.plugins(): + diff_plugins = set(export_sq["plugins"].items()) - set(import_sq.plugins().items()) + if len(diff_plugins) > 0: raise exceptions.UnsupportedOperation( - f"Plugin list is different on the import and export platforms ({str(export_sq['plugins'])} vs {str(import_sq.plugins())}), aborting..." + f"Export platform has the following plugins ({str(diff_plugins)}) missing in the import platform, aborting..." ) From 302d51c8b70a9754cd0177f8da761b325588c4ad Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Thu, 13 Feb 2025 17:13:40 +0100 Subject: [PATCH 07/24] Fixes #1561 (#1573) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4c91eb120..d688a153c 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ deletes tokens created since more than a certain number of days - [Release notes](https://github.com/okorach/sonar-tools/releases) # Requirements and Installation -- `sonar-tools` requires python 3.6 or higher +- `sonar-tools` requires python 3.8 or higher - Installation is based on [pip](https://pypi.org/project/pip/). - Online installation. - Run: `python3 -m pip install sonar-tools` (or `python3 -m pip upgrade sonar-tools`) From 01179db7f326275b88181b4c21f2cca53811e792 Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sun, 2 Mar 2025 20:04:28 +0100 Subject: [PATCH 08/24] sonar-audit avoid excessive history retention (#1575) * Add audit rule for project history size * Add config setting for history size * Audit project and branch history size * Set maxHistoryCount to 100 * Filter QG, QP and Upgrade events from history count * Add docs * Quality pass --- doc/sonar-audit.md | 4 ++- sonar/audit/rules.json | 6 ++++ sonar/audit/rules.py | 1 + sonar/audit/sonar-audit.properties | 5 ++++ sonar/branches.py | 2 +- sonar/components.py | 47 ++++++++++++++++++++++++++++-- sonar/projects.py | 1 + 7 files changed, 62 insertions(+), 4 deletions(-) diff --git a/doc/sonar-audit.md b/doc/sonar-audit.md index c85d2e961..1e0081796 100644 --- a/doc/sonar-audit.md +++ b/doc/sonar-audit.md @@ -193,7 +193,9 @@ sonar-audit --what projects -f projectsAudit.csv --csvSeparator ';' - Last background task with failed SCM detection - Last background task on main branch `FAILED` - Last analysis with an obsolete scanner version (by default more than 2 years old) - - Project analyzed with apparently a wrong scanner (Can't be certain in all cases) + - Projects analyzed with apparently a wrong scanner (Can't be certain in all cases) + - Projects with too many analysis history data points (due to wrong housekeeping settings + or wrong usage of `sonar.projectVersion`) - Branches: (if `audit.project.branches = yes`, default `yes`) - Branches never analyzed but marked as "keep when inactive" - Portfolios: (if `audit.applications = yes`, default `yes`) diff --git a/sonar/audit/rules.json b/sonar/audit/rules.json index f21ea7d09..db2c4f20d 100644 --- a/sonar/audit/rules.json +++ b/sonar/audit/rules.json @@ -290,6 +290,12 @@ "object": "Project", "message": "{} with {} LoCs is stale, it has not been analyzed since {} days, it could be deleted" }, + "PROJ_HISTORY_COUNT": { + "severity": "HIGH", + "type": "PERFORMANCE", + "object": "Branch", + "message": "{} has {} data points in history, this is excessive and causes excessive DB size" + }, "BRANCH_NEVER_ANALYZED": { "severity": "LOW", "type": "PERFORMANCE", diff --git a/sonar/audit/rules.py b/sonar/audit/rules.py index 3c4fb3e4f..687497213 100644 --- a/sonar/audit/rules.py +++ b/sonar/audit/rules.py @@ -134,6 +134,7 @@ class RuleId(enum.Enum): ANT_SCANNER_DEPRECATED = 1311 PROJ_SCM_UNDETECTED = 1312 PROJ_WRONG_SCANNER = 1313 + PROJ_HISTORY_COUNT = 1314 NOT_USING_BRANCH_ANALYSIS = 1400 SIF_UNDETECTED_SCM = 1401 diff --git a/sonar/audit/sonar-audit.properties b/sonar/audit/sonar-audit.properties index 2cb588c93..9ea9db1fc 100644 --- a/sonar/audit/sonar-audit.properties +++ b/sonar/audit/sonar-audit.properties @@ -173,6 +173,11 @@ audit.projects.suspiciousExclusionsExceptions = \\*\\*/(__pycache__|libs|lib|ven # Set property to 0 to turn off the check audit.projects.maxLastAnalysisAge = 180 +# Audit (and warn) for projects/project branches with too many data points in history +# Set property to 0 to turn off the check +audit.projects.maxHistoryCount = 100 + + # Audit branches for zero LoC and last analysis date audit.projects.branches = yes diff --git a/sonar/branches.py b/sonar/branches.py index 5e2d6aba2..3f0b8216f 100644 --- a/sonar/branches.py +++ b/sonar/branches.py @@ -363,7 +363,6 @@ def audit(self, audit_settings: types.ConfigSettings) -> list[Problem]: :param ConfigSettings audit_settings: Options of what to audit and thresholds to raise problems :return: List of problems found, or empty list - :rtype: list[Problem] """ if not audit_settings.get("audit.project.branches", True): log.debug("Branch audit disabled, skipping audit of %s", str(self)) @@ -375,6 +374,7 @@ def audit(self, audit_settings: types.ConfigSettings) -> list[Problem]: + self.__audit_zero_loc() + self.__audit_never_analyzed() + self._audit_bg_task(audit_settings) + + self._audit_history_retention(audit_settings) ) except Exception as e: log.error("%s while auditing %s, audit skipped", util.error_msg(e), str(self)) diff --git a/sonar/components.py b/sonar/components.py index 80e0ba664..54e46d615 100644 --- a/sonar/components.py +++ b/sonar/components.py @@ -37,7 +37,8 @@ from sonar import settings, tasks, measures, utilities, rules -import sonar.audit.problem as pb +from sonar.audit.rules import get_rule, RuleId +from sonar.audit.problem import Problem # Character forbidden in keys that can be used to separate a key from a post fix KEY_SEPARATOR = " " @@ -247,7 +248,26 @@ def set_visibility(self, visibility: str) -> None: settings.set_visibility(self.endpoint, visibility=visibility, component=self) self._visibility = visibility - def _audit_bg_task(self, audit_settings: types.ConfigSettings) -> list[pb.Problem]: + def get_analyses(self, filter_in: Optional[list[str]] = None, filter_out: Optional[list[str]] = None) -> types.ApiPayload: + """Returns a component analyses""" + data = self.endpoint.get_paginated("project_analyses/search", return_field="analyses", params=self.api_params(c.GET))["analyses"] + if filter_in and len(filter_in) > 0: + data = [d for d in data if any(e["category"] in filter_in for e in d["events"])] + if filter_out and len(filter_out) > 0: + data = [d for d in data if all(e["category"] not in filter_out for e in d["events"])] + log.debug("Component analyses = %s", utilities.json_dump(data)) + return data + + def get_versions(self) -> dict[str, datetime]: + """Returns a dict of project versions and their dates""" + data = { + a["version"]: utilities.string_to_date(a["date"]) + for a in reversed(self.get_analyses(filter_in=["VERSION"], filter_out=["QUALITY_GATE", "QUALITY_PROFILE", "SQ_UPGRADE"])) + } + log.debug("Component versions = %s", utilities.json_dump(data)) + return data + + def _audit_bg_task(self, audit_settings: types.ConfigSettings) -> list[Problem]: """Audits project background tasks""" if audit_settings.get("audit.mode", "") == "housekeeper": return [] @@ -266,6 +286,29 @@ def _audit_bg_task(self, audit_settings: types.ConfigSettings) -> list[pb.Proble return last_task.audit(audit_settings) return [] + def _audit_history_retention(self, audit_settings: types.ConfigSettings) -> list[Problem]: + """Audits whether a project has an excessive number of history data points + + :param dict audit_settings: Options of what to audit and thresholds to raise problems + :return: List of problems found, or empty list + """ + if not audit_settings.get("audit.projects.historyRetention", True): + log.debug("%s: History retention audit disabled, audit skipped", str(self)) + return [] + max_history = audit_settings.get("audit.projects.maxHistoryCount", 100) + if max_history == 0: + log.debug("Auditing %s history retention disabled, skipped...", str(self)) + return [] + log.debug("Auditing %s history retention", str(self)) + history = self.get_analyses(filter_out=["QUALITY_GATE", "QUALITY_PROFILE", "SQ_UPGRADE"]) + log.debug("%s has %d history data points, max allowed = %d", str(self), len(history), max_history) + if not history: + return [] + history_len = len(history) + if history_len > max_history: + return [Problem(get_rule(RuleId.PROJ_HISTORY_COUNT), self, str(self), history_len)] + return [] + def last_task(self) -> Optional[tasks.Task]: """Returns the last analysis background task of a problem, or none if not found""" return tasks.search_last(component_key=self.key, endpoint=self.endpoint) diff --git a/sonar/projects.py b/sonar/projects.py index badc58774..f9773d13a 100644 --- a/sonar/projects.py +++ b/sonar/projects.py @@ -684,6 +684,7 @@ def audit(self, audit_settings: types.ConfigSettings, write_q: Queue[list[Proble problems += self._audit_bg_task(audit_settings) problems += self.__audit_binding_valid(audit_settings) problems += self.__audit_scanner(audit_settings) + problems += self._audit_history_retention(audit_settings) except (ConnectionError, RequestException) as e: util.handle_error(e, f"auditing {str(self)}", catch_all=True) From 090f49e039fe38f8ea6430ef6b7557e2e4e155b7 Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sun, 2 Mar 2025 20:06:15 +0100 Subject: [PATCH 09/24] QG tests (#1565) * Fixes * QG tests * Fix ObjectAlreadyExist in create * Fix default SW when deleting TEMP QG * Upgrade alpine base version * Fixes * Fix bug * get_empty_qg() * Change QG fixtures names * Add SONAR_WAY constant * Quality pass * Fix bugs * Formatting * Fix test * Add permissions tests (#1567) --- migration/release.Dockerfile | 2 +- migration/snapshot.Dockerfile | 2 +- sonar/qualitygates.py | 62 ++++++------ test/unit/conftest.py | 40 +++++++- test/unit/test_migration.py | 2 +- test/unit/test_projects.py | 11 +++ test/unit/test_qg.py | 175 ++++++++++++++++++++++++++++++++++ test/unit/test_qp.py | 10 +- test/unit/utilities.py | 2 + 9 files changed, 266 insertions(+), 40 deletions(-) create mode 100644 test/unit/test_qg.py diff --git a/migration/release.Dockerfile b/migration/release.Dockerfile index 1798b0f61..4c813756a 100644 --- a/migration/release.Dockerfile +++ b/migration/release.Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.20.3 +FROM alpine:3.21.2 LABEL maintainer="olivier.korach@gmail.com" ENV IN_DOCKER="Yes" diff --git a/migration/snapshot.Dockerfile b/migration/snapshot.Dockerfile index e4e5283a0..7a83ba9d6 100644 --- a/migration/snapshot.Dockerfile +++ b/migration/snapshot.Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.20.3 +FROM alpine:3.21.2 LABEL maintainer="olivier.korach@gmail.com" ENV IN_DOCKER="Yes" diff --git a/sonar/qualitygates.py b/sonar/qualitygates.py index 8e1391461..a2eb44bce 100644 --- a/sonar/qualitygates.py +++ b/sonar/qualitygates.py @@ -51,6 +51,10 @@ "new_coverage": (20, 90, "Coverage below 20% is a too low bar, above 90% is overkill"), "new_bugs": (0, 0, __NEW_ISSUES_SHOULD_BE_ZERO), "new_vulnerabilities": (0, 0, __NEW_ISSUES_SHOULD_BE_ZERO), + "new_violations": (0, 0, __NEW_ISSUES_SHOULD_BE_ZERO), + "new_software_quality_blocker_issues": (0, 0, __NEW_ISSUES_SHOULD_BE_ZERO), + "new_software_quality_high_issues": (0, 0, __NEW_ISSUES_SHOULD_BE_ZERO), + "new_software_quality_medium_issues": (0, 0, __NEW_ISSUES_SHOULD_BE_ZERO), "new_security_hotspots": (0, 0, __NEW_ISSUES_SHOULD_BE_ZERO), "new_blocker_violations": (0, 0, __NEW_ISSUES_SHOULD_BE_ZERO), "new_critical_violations": (0, 0, __NEW_ISSUES_SHOULD_BE_ZERO), @@ -101,10 +105,9 @@ def __init__(self, endpoint: pf.Platform, name: str, data: types.ApiPayload) -> def get_object(cls, endpoint: pf.Platform, name: str) -> QualityGate: """Reads a quality gate from SonarQube - :param Platform endpoint: Reference to the SonarQube platform - :param str name: Quality gate + :param endpoint: Reference to the SonarQube platform + :param name: Quality gate :return: the QualityGate object or None if not found - :rtype: QualityGate or None """ o = QualityGate.CACHE.get(name, endpoint.url) if o: @@ -118,21 +121,25 @@ def get_object(cls, endpoint: pf.Platform, name: str) -> QualityGate: def load(cls, endpoint: pf.Platform, data: types.ApiPayload) -> QualityGate: """Creates a quality gate from returned API data :return: the QualityGate object - :rtype: QualityGate or None """ # SonarQube 10 compatibility: "id" field dropped, replaced by "name" o = QualityGate.CACHE.get(data["name"], endpoint.url) if not o: o = cls(endpoint, data["name"], data=data) + log.debug("Loading 2 %s QG from %s", o.name, util.json_dump(data)) o.sq_json = data + o.is_default = data.get("isDefault", False) + o.is_built_in = data.get("isBuiltIn", False) return o @classmethod def create(cls, endpoint: pf.Platform, name: str) -> Union[QualityGate, None]: """Creates an empty quality gate""" - r = endpoint.post(QualityGate.API[c.CREATE], params={"name": name}) - if not r.ok: - return None + try: + endpoint.post(QualityGate.API[c.CREATE], params={"name": name}) + except (ConnectionError, RequestException) as e: + util.handle_error(e, f"creating quality gate '{name}'", catch_http_errors=(HTTPStatus.BAD_REQUEST,)) + raise exceptions.ObjectAlreadyExists(name, e.response.text) return cls.get_object(endpoint, name) def __str__(self) -> str: @@ -179,13 +186,6 @@ def projects(self) -> dict[str, projects.Project]: page += 1 return self._projects - def count_projects(self) -> int: - """ - :return: The number of projects using this quality gate - :rtype: int - """ - return len(self.projects()) - def conditions(self, encoded: bool = False) -> list[str]: """ :param encoded: Whether to encode the conditions or not, defaults to False @@ -202,24 +202,23 @@ def conditions(self, encoded: bool = False) -> list[str]: return _encode_conditions(self._conditions) return self._conditions - def clear_conditions(self) -> None: + def clear_conditions(self) -> bool: """Clears all quality gate conditions, if quality gate is not built-in :return: Nothing """ if self.is_built_in: log.debug("Can't clear conditions of built-in %s", str(self)) - else: - log.debug("Clearing conditions of %s", str(self)) - for cond in self.conditions(): - self.post("qualitygates/delete_condition", params={"id": cond["id"]}) - self._conditions = None + return False + log.debug("Clearing conditions of %s", str(self)) + for cond in self.conditions(): + self.post("qualitygates/delete_condition", params={"id": cond["id"]}) + self._conditions = [] + return True def set_conditions(self, conditions_list: list[str]) -> bool: """Sets quality gate conditions (overriding any previous conditions) as encoded in sonar-config - :param conditions_list: List of conditions, encoded - :type conditions_list: dict + :param list[str] conditions_list: List of conditions, encoded :return: Whether the operation succeeded - :rtype: bool """ if not conditions_list or len(conditions_list) == 0: return True @@ -236,6 +235,7 @@ def set_conditions(self, conditions_list: list[str]) -> bool: for cond in conditions_list: (params["metric"], params["op"], params["error"]) = _decode_condition(cond) ok = ok and self.post("qualitygates/create_condition", params=params).ok + self._conditions = None self.conditions() return ok @@ -248,12 +248,11 @@ def permissions(self) -> permissions.QualityGatePermissions: self._permissions = permissions.QualityGatePermissions(self) return self._permissions - def set_permissions(self, permissions_list: types.ObjectJsonRepr) -> QualityGate: + def set_permissions(self, permissions_list: types.ObjectJsonRepr) -> bool: """Sets quality gate permissions :param permissions_list: :type permissions_list: dict {"users": [], "groups": []} :return: Whether the operation succeeded - :rtype: bool """ return self.permissions().set(permissions_list) @@ -331,7 +330,6 @@ def audit(self, audit_settings: types.ConfigSettings = None) -> list[Problem]: elif nb_conditions > max_cond: problems.append(Problem(get_rule(RuleId.QG_TOO_MANY_COND), self, my_name, nb_conditions, max_cond)) problems += self.__audit_conditions() - log.debug("Auditing that %s has some assigned projects", my_name) if not self.is_default and len(self.projects()) == 0: problems.append(Problem(get_rule(RuleId.QG_NOT_USED), self, my_name)) return problems @@ -341,13 +339,13 @@ def to_json(self, export_settings: types.ConfigSettings) -> types.ObjectJsonRepr json_data = self.sq_json full = export_settings.get("FULL_EXPORT", False) if not self.is_default and not full: - json_data.pop("isDefault") + json_data.pop("isDefault", None) if self.is_built_in: if full: json_data["_conditions"] = self.conditions(encoded=True) else: if not full: - json_data.pop("isBuiltIn") + json_data.pop("isBuiltIn", None) json_data["conditions"] = self.conditions(encoded=True) json_data["permissions"] = self.permissions().export(export_settings=export_settings) return util.remove_nones(util.filter_export(json_data, _IMPORTABLE_PROPERTIES, full)) @@ -382,10 +380,14 @@ def get_list(endpoint: pf.Platform) -> dict[str, QualityGate]: data = json.loads(endpoint.get(QualityGate.API[c.LIST]).text) qg_list = {} for qg in data["qualitygates"]: - log.debug("Getting QG %s", util.json_dump(qg)) - qg_obj = QualityGate(endpoint=endpoint, name=qg["name"], data=qg.copy()) + qg_obj = QualityGate.CACHE.get(qg["name"], endpoint.url) + if qg_obj is None: + qg_obj = QualityGate(endpoint=endpoint, name=qg["name"], data=qg.copy()) if endpoint.version() < (7, 9, 0) and "default" in data and data["default"] == qg["id"]: qg_obj.is_default = True + else: + qg_obj.is_default = qg.get("isDefault", False) + qg_obj.is_built_in = qg.get("isBuiltIn", False) qg_list[qg_obj.name] = qg_obj return dict(sorted(qg_list.items())) diff --git a/test/unit/conftest.py b/test/unit/conftest.py index 898bb95db..3bf8459f0 100644 --- a/test/unit/conftest.py +++ b/test/unit/conftest.py @@ -66,6 +66,42 @@ def get_test_project() -> Generator[projects.Project]: pass +@pytest.fixture +def get_empty_qg() -> Generator[qualitygates.QualityGate]: + """setup of tests""" + try: + o = qualitygates.QualityGate.get_object(endpoint=util.SQ, name=util.TEMP_KEY) + except exceptions.ObjectNotFound: + o = qualitygates.QualityGate.create(endpoint=util.SQ, name=util.TEMP_KEY) + o.clear_conditions() + yield o + # Teardown: Clean up resources (if any) after the test + o.key = util.TEMP_KEY + try: + sw = qualitygates.QualityGate.get_object(endpoint=util.SQ, name=util.SONAR_WAY) + sw.set_as_default() + o.delete() + except exceptions.ObjectNotFound: + pass + + +@pytest.fixture +def get_loaded_qg() -> Generator[qualitygates.QualityGate]: + """setup of tests""" + try: + o = qualitygates.QualityGate.get_object(endpoint=util.SQ, name=util.TEMP_KEY) + except exceptions.ObjectNotFound: + o = qualitygates.QualityGate.create(endpoint=util.SQ, name=util.TEMP_KEY) + yield o + o.key = util.TEMP_KEY + try: + sw = qualitygates.QualityGate.get_object(endpoint=util.SQ, name=util.SONAR_WAY) + sw.set_as_default() + o.delete() + except exceptions.ObjectNotFound: + pass + + @pytest.fixture def get_test_app() -> Generator[applications.Application]: """setup of tests""" @@ -139,7 +175,7 @@ def get_test_qp() -> Generator[qualityprofiles.QualityProfile]: try: o = qualityprofiles.get_object(endpoint=util.SQ, name=util.TEMP_KEY, language="py") if o.is_default: - sw = qualityprofiles.get_object(endpoint=util.SQ, name="Sonar way", language="py") + sw = qualityprofiles.get_object(endpoint=util.SQ, name=util.SONAR_WAY, language="py") sw.set_as_default() except exceptions.ObjectNotFound: o = qualityprofiles.QualityProfile.create(endpoint=util.SQ, name=util.TEMP_KEY, language="py") @@ -231,7 +267,7 @@ def get_sarif_file() -> Generator[str]: def get_test_quality_gate() -> Generator[qualitygates.QualityGate]: """setup of tests""" util.start_logging() - sonar_way = qualitygates.QualityGate.get_object(util.SQ, "Sonar way") + sonar_way = qualitygates.QualityGate.get_object(util.SQ, util.SONAR_WAY) o = sonar_way.copy(util.TEMP_KEY) yield o try: diff --git a/test/unit/test_migration.py b/test/unit/test_migration.py index 1ef138a2d..05bf72c04 100644 --- a/test/unit/test_migration.py +++ b/test/unit/test_migration.py @@ -104,7 +104,7 @@ def test_migration(get_json_file: Generator[str]) -> None: assert json_config["projects"]["demo:gitlab-ci-maven"]["detectedCi"] == "Gitlab CI" assert json_config["projects"]["demo:github-actions-cli"]["detectedCi"] == "Github Actions" if util.SQ.edition() != "community": - assert len(p["branches"]["main"]["issues"]["thirdParty"]) > 0 + assert p["branches"]["main"]["issues"]["thirdParty"] > 0 for p in json_config["portfolios"].values(): assert "projects" in p diff --git a/test/unit/test_projects.py b/test/unit/test_projects.py index 1caae387b..432273dcd 100644 --- a/test/unit/test_projects.py +++ b/test/unit/test_projects.py @@ -257,3 +257,14 @@ def test_wrong_key_2(get_test_project: Generator[projects.Project]) -> None: # assert proj.quality_gate() is None with pytest.raises(exceptions.ObjectNotFound): proj.audit({}, None) + +def test_set_permissions(get_test_project: Generator[projects.Project]) -> None: + """test_set_permissions""" + proj = get_test_project + perms = proj.permissions().to_json() + assert "tech-leads" in perms["groups"] + proj.set_permissions({"users": {"admin": ["user", "admin"], "olivier": ["user"]}}) + perms = proj.permissions().to_json() + assert "groups" not in perms + assert len(perms["users"]) == 2 + diff --git a/test/unit/test_qg.py b/test/unit/test_qg.py new file mode 100644 index 000000000..d38fe61dc --- /dev/null +++ b/test/unit/test_qg.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +# +# sonar-tools tests +# Copyright (C) 2025 Olivier Korach +# mailto:olivier.korach AT gmail DOT com +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3 of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# + +""" quality gates tests """ + +from collections.abc import Generator +import json +import pytest + +import utilities as util +from sonar import qualitygates, exceptions, logging + + +def test_get_object(get_loaded_qg: Generator[qualitygates.QualityGate]) -> None: + """Test get_object and verify that if requested twice the same object is returned""" + qg = get_loaded_qg + assert qg.name == util.TEMP_KEY + assert str(qg) == f"quality gate '{util.TEMP_KEY}'" + assert qg.url() == f"{util.SQ.url}/quality_gates/show/{util.TEMP_KEY}" + qg2 = qualitygates.QualityGate.get_object(endpoint=util.SQ, name=util.TEMP_KEY) + assert qg.projects() == {} + assert qg.projects() == {} + assert qg2 is qg + + +def test_get_object_non_existing() -> None: + """Test exception raised when providing non existing portfolio key""" + + with pytest.raises(exceptions.ObjectNotFound) as e: + _ = qualitygates.QualityGate.get_object(endpoint=util.SQ, name=util.NON_EXISTING_KEY) + assert str(e.value).endswith(f"Quality gate '{util.NON_EXISTING_KEY}' not found") + + +def test_exists(get_loaded_qg: Generator[qualitygates.QualityGate]) -> None: + """Test exist""" + _ = get_loaded_qg + assert qualitygates.exists(endpoint=util.SQ, gate_name=util.TEMP_KEY) + assert not qualitygates.exists(endpoint=util.SQ, gate_name=util.NON_EXISTING_KEY) + + +def test_get_list() -> None: + """Test QP get_list""" + qgs = qualitygates.get_list(endpoint=util.SQ) + assert len(qgs) >= 5 + + +def test_create_delete(get_loaded_qg: Generator[qualitygates.QualityGate]) -> None: + """Test QG create delete""" + qp = get_loaded_qg + assert qp is not None + + with pytest.raises(exceptions.ObjectAlreadyExists): + qualitygates.QualityGate.create(endpoint=util.SQ, name=util.TEMP_KEY) + qp.delete() + assert not qualitygates.exists(endpoint=util.SQ, gate_name=util.TEMP_KEY) + + +def test_set_conditions(get_loaded_qg: Generator[qualitygates.QualityGate]) -> None: + """test_set_conditions""" + qg = get_loaded_qg + sw = qualitygates.QualityGate.get_object(util.SQ, util.SONAR_WAY) + assert sorted(qg.conditions(encoded=True)) == sorted(sw.conditions(encoded=True)) + qg.clear_conditions() + assert qg.set_conditions(None) + assert qg.set_conditions([]) + assert qg.set_conditions(["new_coverage <= 80"]) + assert qg.conditions(encoded=True) == ["new_coverage <= 80"] + qg.clear_conditions() + assert qg.conditions() == [] + assert qg.set_conditions(["new_coverage <= 80"]) + assert qg.conditions(encoded=True) == ["new_coverage <= 80"] + assert qg.set_conditions(["new_coverage <= 50", "new_violations >= 0", "test_success_density <= 100"]) + assert qg.conditions(encoded=True) == ["new_coverage <= 50", "new_violations >= 0", "test_success_density <= 100"] + + +def test_clear_conditions(get_loaded_qg: Generator[qualitygates.QualityGate]) -> None: + """test_clear_conditions""" + sw = qualitygates.QualityGate.get_object(util.SQ, util.SONAR_WAY) + assert not sw.clear_conditions() + assert len(sw.conditions()) >= 3 + + qg = get_loaded_qg + assert len(qg.conditions()) >= 3 + assert qg.clear_conditions() + assert len(qg.conditions()) == 0 + assert qg.conditions() == [] + + +def test_permissions(get_loaded_qg: Generator[qualitygates.QualityGate]) -> None: + """test_permissions""" + qg = get_loaded_qg + assert qg.set_permissions({"users": ["olivier", "michal"]}) + + +def test_copy(get_loaded_qg: Generator[qualitygates.QualityGate]) -> None: + """test_copy""" + qg = get_loaded_qg + assert qg.set_conditions(["new_coverage <= 50", "new_violations >= 0", "test_success_density <= 100"]) + + qg2 = qg.copy("TEMP_NAME2") + assert qg2.conditions(encoded=True) == ["new_coverage <= 50", "new_violations >= 0", "test_success_density <= 100"] + qg2.delete() + + +def test_set_as_default(get_loaded_qg: Generator[qualitygates.QualityGate]) -> None: + """test_set_as_default""" + qg = get_loaded_qg + sw = qualitygates.QualityGate.get_object(util.SQ, util.SONAR_WAY) + assert sw.is_built_in + qg.set_conditions(["new_coverage <= 50", "new_violations >= 0", "test_success_density <= 100"]) + assert qg.set_as_default() + assert not sw.is_default + assert sw.is_built_in + + +def test_audit(get_empty_qg: Generator[qualitygates.QualityGate]) -> None: + """test_audit""" + qg = get_empty_qg + for pb in qg.audit(): + logging.debug(str(pb)) + assert len(qg.audit()) == 2 + qg.set_conditions(["new_coverage <= 50", "new_duplicated_lines_density >= 3"]) + assert len(qg.audit()) == 1 + conds = [ + "new_coverage <= 80", + "new_duplicated_lines_density >= 3", + "new_security_hotspots_reviewed <= 100", + "new_software_quality_blocker_issues >= 0", + "new_software_quality_high_issues >= 0", + "new_software_quality_medium_issues >= 0", + ] + qg.set_conditions(conds) + assert len(qg.audit({"audit.qualitygates.maxConditions": 5})) == 2 + + +def test_count(): + count = qualitygates.count(util.SQ) + assert count >= 5 + qg = qualitygates.QualityGate.create(util.SQ, util.TEMP_KEY) + assert qualitygates.count(util.SQ) == count + 1 + qg.delete() + assert qualitygates.count(util.SQ) == count + + +def test_export() -> None: + """test_export""" + json_exp = qualitygates.export(endpoint=util.SQ, export_settings={}) + yaml_exp = qualitygates.convert_for_yaml(json_exp) + assert len(json_exp) > 0 + assert isinstance(json_exp, dict) + assert isinstance(yaml_exp, list) + assert len(yaml_exp) == len(json_exp) + + +def test_audit_disabled() -> None: + """test_audit_disabled""" + assert len(qualitygates.audit(util.SQ, {"audit.qualityGates": False})) == 0 diff --git a/test/unit/test_qp.py b/test/unit/test_qp.py index 9ccc62d26..3c419c101 100644 --- a/test/unit/test_qp.py +++ b/test/unit/test_qp.py @@ -75,13 +75,13 @@ def test_create_delete(get_test_qp: Generator[qualityprofiles.QualityProfile]) - def test_inheritance(get_test_qp: Generator[qualityprofiles.QualityProfile]) -> None: """Test addition of a project in manual mode""" qp = get_test_qp - sonar_way_qp = qualityprofiles.get_object(util.SQ, "Sonar way", "py") + sonar_way_qp = qualityprofiles.get_object(util.SQ, util.SONAR_WAY, "py") assert not qp.is_child() - assert qp.set_parent("Sonar way") + assert qp.set_parent(util.SONAR_WAY) assert qp.is_child() assert qp.inherits_from_built_in() - assert qp.parent_name == "Sonar way" + assert qp.parent_name == util.SONAR_WAY assert qp.built_in_parent() is sonar_way_qp with pytest.raises(exceptions.ObjectNotFound) as e: @@ -109,7 +109,7 @@ def test_set_default(get_test_qp: Generator[qualityprofiles.QualityProfile]) -> assert not qp.is_default assert qp.set_as_default() assert qp.is_default - sonar_way_qp = qualityprofiles.get_object(util.SQ, "Sonar way", "py") + sonar_way_qp = qualityprofiles.get_object(util.SQ, util.SONAR_WAY, "py") assert sonar_way_qp.set_as_default() assert sonar_way_qp.is_default assert not qp.is_default @@ -146,7 +146,7 @@ def test_add_remove_rules(get_test_qp: Generator[qualityprofiles.QualityProfile] qp.deactivate_rules([RULE1, RULE2]) assert len(qp.rules()) == 1 - assert qp.set_parent("Sonar way") + assert qp.set_parent(util.SONAR_WAY) rulecount = len(qp.rules()) assert rulecount > 250 if util.SQ.version() >= (10, 0, 0) else 200 diff --git a/test/unit/utilities.py b/test/unit/utilities.py index 095eef5fd..964a30b64 100644 --- a/test/unit/utilities.py +++ b/test/unit/utilities.py @@ -81,6 +81,8 @@ TAGS = ["foo", "bar"] +SONAR_WAY = "Sonar way" + def clean(*files: str) -> None: """Deletes a list of file if they exists""" From 68bc4eb05f80dd7560ec3b4552f75c845412c390 Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Mon, 3 Mar 2025 10:47:40 +0100 Subject: [PATCH 10/24] Fix trivy unknown severities (#1577) * Fixes case where trivy severity is unknown * Formatting * Fix scan process --- conf/scan.sh | 10 ++++------ conf/trivy2sonar.py | 2 ++ test/unit/test_projects.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/conf/scan.sh b/conf/scan.sh index c107771d1..594ce7252 100755 --- a/conf/scan.sh +++ b/conf/scan.sh @@ -42,6 +42,9 @@ do -test) dotest="true" ;; + -local) + localbuild="true" + ;; *) scanOpts=("${scanOpts[@]}" "$1") ;; @@ -54,7 +57,7 @@ pylintReport="$buildDir/pylint-report.out" flake8Report="$buildDir/flake8-report.out" [ ! -d "$buildDir" ] && mkdir "$buildDir" -rm -rf -- ${buildDir:?"."}/* .coverage */__pycache__ */*.pyc # mediatools/__pycache__ testpytest/__pycache__ testunittest/__pycache__ +# rm -rf -- ${buildDir:?"."}/* .coverage */__pycache__ */*.pyc # mediatools/__pycache__ testpytest/__pycache__ testunittest/__pycache__ if [ "$dolint" != "false" ]; then @@ -101,8 +104,3 @@ echo "Running: $cmd" | sed "s/$SONAR_TOKEN//g" echo $cmd - -for target in lts latest -do - rm -rf "$ROOTDIR/test/$target" -done \ No newline at end of file diff --git a/conf/trivy2sonar.py b/conf/trivy2sonar.py index 50606789c..4d8ec0808 100755 --- a/conf/trivy2sonar.py +++ b/conf/trivy2sonar.py @@ -63,6 +63,8 @@ def main() -> None: # else: # sev = "HIGH" sev_mqr = issue.get("Severity", "MEDIUM") + if sev_mqr == "UNKNOWN": + sev_mqr = "MEDIUM" rules_dict[f"{TOOLNAME}:{issue['VulnerabilityID']}"] = { "id": f"{TOOLNAME}:{issue['VulnerabilityID']}", "name": f"{TOOLNAME}:{issue['VulnerabilityID']} - {issue['Title']}", diff --git a/test/unit/test_projects.py b/test/unit/test_projects.py index 432273dcd..1742b1955 100644 --- a/test/unit/test_projects.py +++ b/test/unit/test_projects.py @@ -258,6 +258,7 @@ def test_wrong_key_2(get_test_project: Generator[projects.Project]) -> None: with pytest.raises(exceptions.ObjectNotFound): proj.audit({}, None) + def test_set_permissions(get_test_project: Generator[projects.Project]) -> None: """test_set_permissions""" proj = get_test_project @@ -267,4 +268,3 @@ def test_set_permissions(get_test_project: Generator[projects.Project]) -> None: perms = proj.permissions().to_json() assert "groups" not in perms assert len(perms["users"]) == 2 - From 9b4200a57ac382955e235d96c9e1c40f67ded1a8 Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Mon, 3 Mar 2025 13:18:38 +0100 Subject: [PATCH 11/24] Fixes #1574 (#1578) --- README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d688a153c..d6d0b0681 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,10 @@ # sonar-tools -Command line tools to help in SonarQube administration tasks. Available as a **pypi.org** package or a **docker** image +Command line tools to help in SonarQube administration tasks. Available as a **pypi.org** package or a **docker** image. + +`sonar-tools` is compatible with all SonarQube versions starting from 9.9 up to latest 2025.1. +It way work with older versions but this is not guaranteed. +Compatibility with recent SonarQube Community Builds should be fine, although it has not been formally validated yet. + ![Downloads](https://img.shields.io/pypi/dm/sonar-tools?color=informational) ![Python-Versions](https://img.shields.io/pypi/pyversions/sonar-tools) @@ -12,7 +17,7 @@ Command line tools to help in SonarQube administration tasks. Available as a **p [![Bugs](https://sonarcloud.io/api/project_badges/measure?project=okorach_sonar-tools&metric=bugs)](https://sonarcloud.io/dashboard?id=okorach_sonar-tools) [![Lines of Code](https://sonarcloud.io/api/project_badges/measure?project=okorach_sonar-tools&metric=ncloc)](https://sonarcloud.io/dashboard?id=okorach_sonar-tools) -**DISCLAIMER**: This software is community software. None of the tools it contains are neither supported nor endorsed by SonarSource S.A. Switzerland, the company publishing the [SonarQube](https://www.sonarqube.org/), [SonarCloud](https://sonarcloud.io) and [SonarLint](https://sonarlint.org) products +**DISCLAIMER**: This software is community software. None of the tools it contains are neither supported nor endorsed by SonarSource S.A. Switzerland, the company publishing the [SonarQube Server](https://www.sonarsource.com/products/sonarqube/), [SonarQube Cloud](https://sonarcloud.io) and [SonarQube for IDE (ex- SonarLint](https://www.sonarsource.com/products/sonarlint/) products The following utilities are available: - [sonar-audit](#sonar-audit): Audits a SonarQube instance, and reports all the problems From 98e2b23cf0cceec2f8d143e3da0b119e26aeafcc Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sat, 8 Mar 2025 13:21:25 +0100 Subject: [PATCH 12/24] Adapt tests for 2025.1 (#1581) * Fix cache on QP * Adapt test for less restrictive project move criteria * Adjust to sonar-tools smaller number of issues * Adapt to issue whose changelog size was increased * REmove default QP before delete * Fix requirements to build. Add pytest and coverage * Add new AI code assurance values for 25.1 * Move get_ai_code_assurance in component since it can apply to branches * Adapt for AI code * Fix update login API v2 * Fix user update with PATCH * Fix flaky test dur to user delete not possible * Fix 9.9 tests * Add hotspot tests * Fix search by directory bug * Show docker container creation command * Fix tests * Formatting --- requirements-to-build.txt | 2 ++ sonar/components.py | 21 ++++++++++++++- sonar/issues.py | 4 ++- sonar/projects.py | 17 ++++-------- sonar/qualityprofiles.py | 1 + sonar/users.py | 54 ++++++++++++++++++++++---------------- test/integration/it.sh | 2 +- test/unit/test_cli.py | 29 ++++++++++---------- test/unit/test_findings.py | 4 +-- test/unit/test_hotspots.py | 37 ++++++++++++++++++++++++++ test/unit/test_issues.py | 16 +++++------ test/unit/test_projects.py | 18 ++++++++++--- test/unit/test_qp.py | 10 +++---- test/unit/test_users.py | 15 ++++++----- 14 files changed, 152 insertions(+), 78 deletions(-) create mode 100644 test/unit/test_hotspots.py diff --git a/requirements-to-build.txt b/requirements-to-build.txt index 16ec9ee14..764491623 100644 --- a/requirements-to-build.txt +++ b/requirements-to-build.txt @@ -5,3 +5,5 @@ sphinx sphinx_rtd_theme sphinx-autodoc-typehints twine +pytest +coverage diff --git a/sonar/components.py b/sonar/components.py index 54e46d615..236bd44d0 100644 --- a/sonar/components.py +++ b/sonar/components.py @@ -28,6 +28,7 @@ import json from datetime import datetime +from requests import RequestException from sonar.util import types import sonar.util.constants as c @@ -35,7 +36,7 @@ import sonar.sqobject as sq import sonar.platform as pf -from sonar import settings, tasks, measures, utilities, rules +from sonar import settings, tasks, measures, utilities, rules, exceptions from sonar.audit.rules import get_rule, RuleId from sonar.audit.problem import Problem @@ -267,6 +268,24 @@ def get_versions(self) -> dict[str, datetime]: log.debug("Component versions = %s", utilities.json_dump(data)) return data + def get_ai_code_assurance(self) -> Optional[str]: + """ + :return: The AI code assurance status of a project or a branch + """ + log.debug("AI Code assurance version = %s", str(self.endpoint.version())) + api = "project/get_ai_code_assurance" + if self.endpoint.version() >= (2025, 1, 0): + api = "project_branches/get_ai_code_assurance" + try: + return str(json.loads(self.get(api, params=self.api_params(c.GET)).text)["aiCodeAssurance"]).upper() + except (ConnectionError, RequestException) as e: + utilities.handle_error(e, f"getting AI code assurance of {str(self)}", catch_all=True) + if "Unknown url" in str(e): + raise exceptions.UnsupportedOperation( + f"AI code assurance is not available for {self.endpoint.edition()} edition version {str(self.endpoint.version())}" + ) + return None + def _audit_bg_task(self, audit_settings: types.ConfigSettings) -> list[Problem]: """Audits project background tasks""" if audit_settings.get("audit.mode", "") == "housekeeper": diff --git a/sonar/issues.py b/sonar/issues.py index aa26ddde2..24ba8b2de 100644 --- a/sonar/issues.py +++ b/sonar/issues.py @@ -577,9 +577,11 @@ def component_filter(endpoint: pf.Platform) -> str: def search_by_directory(endpoint: pf.Platform, params: ApiParams) -> dict[str, Issue]: """Searches issues splitting by directory to avoid exceeding the 10K limit""" new_params = params.copy() + if "components" in params: + new_params[component_filter(endpoint)] = params["components"] + log.info("Splitting search by directories with %s", util.json_dump(new_params)) facets = _get_facets(endpoint=endpoint, project_key=new_params[component_filter(endpoint)], facets="directories", params=new_params) issue_list = {} - log.info("Splitting search by directories") for d in facets["directories"]: new_params["directories"] = d["val"] issue_list.update(search(endpoint=endpoint, params=new_params, raise_error=True)) diff --git a/sonar/projects.py b/sonar/projects.py index f9773d13a..dff755b68 100644 --- a/sonar/projects.py +++ b/sonar/projects.py @@ -1095,9 +1095,13 @@ def export(self, export_settings: types.ConfigSettings, settings_list: dict[str, settings_dict = settings.get_bulk(endpoint=self.endpoint, component=self, settings_list=settings_list, include_not_set=False) # json_data.update({s.to_json() for s in settings_dict.values() if include_inherited or not s.inherited}) - if self.endpoint.version() >= (10, 7, 0) and self.endpoint.edition() != "community": + contains_ai = False + try: ai = self.get_ai_code_assurance() contains_ai = ai is not None and ai != "NONE" + except exceptions.UnsupportedOperation: + pass + if contains_ai: json_data[_CONTAINS_AI_CODE] = contains_ai for s in settings_dict.values(): if not export_settings.get("INCLUDE_INHERITED", False) and s.inherited: @@ -1200,17 +1204,6 @@ def set_contains_ai_code(self, contains_ai_code: bool) -> bool: util.handle_error(e, f"setting contains AI code of {str(self)}", catch_all=True) return False - def get_ai_code_assurance(self) -> Optional[str]: - """ - :return: The AI code assurance status of the project - """ - if self.endpoint.version() >= (10, 7, 0) and self.endpoint.edition() != "community": - try: - return str(json.loads(self.get("projects/get_ai_code_assurance", params={"project": self.key}).text)["aiCodeAssurance"]).upper() - except (ConnectionError, RequestException) as e: - util.handle_error(e, f"getting AI code assurance of {str(self)}", catch_all=True) - return None - def set_quality_profile(self, language: str, quality_profile: str) -> bool: """Sets project quality profile for a given language diff --git a/sonar/qualityprofiles.py b/sonar/qualityprofiles.py index 3eec17a3d..5b54c768b 100644 --- a/sonar/qualityprofiles.py +++ b/sonar/qualityprofiles.py @@ -602,6 +602,7 @@ def get_list(endpoint: pf.Platform, use_cache: bool = True) -> dict[str, Quality with _CLASS_LOCK: if len(QualityProfile.CACHE) == 0 or not use_cache: + QualityProfile.CACHE.clear() search(endpoint=endpoint) return QualityProfile.CACHE.objects diff --git a/sonar/users.py b/sonar/users.py index 5b43a2600..0b699e297 100644 --- a/sonar/users.py +++ b/sonar/users.py @@ -60,7 +60,7 @@ class User(sqobject.SqObject): c.DELETE: USER_API, c.SEARCH: USER_API, "GROUP_MEMBERSHIPS": "v2/authorizations/group-memberships", - "UPDATE_LOGIN": "users/update_login", + "UPDATE_LOGIN": USER_API, } API_V1 = { c.CREATE: "users/create", @@ -277,27 +277,34 @@ def update(self, **kwargs) -> User: log.debug("Updating %s with %s", str(self), str(kwargs)) params = self.api_params(c.UPDATE) my_data = vars(self) - if self.is_local: - params.update({k: kwargs[k] for k in ("name", "email") if k in kwargs and kwargs[k] != my_data[k]}) - if len(params) >= 1: - self.post(User.api_for(c.UPDATE, self.endpoint), params=params) - if "name" in params: - self.name = kwargs["name"] - if "email" in params: - self.email = kwargs["email"] - if "scmAccounts" in kwargs: - self.set_scm_accounts(kwargs["scmAccounts"]) - if "login" in kwargs: - new_login = kwargs["login"] - o = User.CACHE.get(new_login, self.endpoint.url) - if not o: - self.post( - User.api_for("UPDATE_LOGIN", self.endpoint), params={**self.api_params(User.API["UPDATE_LOGIN"]), "newLogin": new_login} - ) - User.CACHE.pop(self) - self.login = new_login - User.CACHE.put(self) self.set_groups(util.csv_to_list(kwargs.get("groups", ""))) + if not self.is_local: + return self + params.update({k: kwargs[k] for k in ("name", "email") if k in kwargs and kwargs[k] != my_data[k]}) + if len(params) >= 1: + api = User.api_for(c.UPDATE, self.endpoint) + if self.endpoint.version() >= (10, 4, 0): + self.patch(f"{api}/{self.id}", params=params) + else: + self.post(api, params=params) + if "name" in params: + self.name = kwargs["name"] + if "email" in params: + self.email = kwargs["email"] + if "scmAccounts" in kwargs: + self.set_scm_accounts(kwargs["scmAccounts"]) + if "login" in kwargs: + new_login = kwargs["login"] + o = User.CACHE.get(new_login, self.endpoint.url) + if not o: + api = User.api_for("UPDATE_LOGIN", self.endpoint) + if self.endpoint.version() >= (10, 4, 0): + self.patch(f"{api}/{self.id}", params={"login": new_login}) + else: + self.post(api, params={**self.api_params(User.API["UPDATE_LOGIN"]), "newLogin": new_login}) + User.CACHE.pop(self) + self.login = new_login + User.CACHE.put(self) return self def add_to_group(self, group_name: str) -> bool: @@ -407,12 +414,13 @@ def set_scm_accounts(self, accounts_list: list[str]) -> bool: :rtype: bool """ log.debug("Setting SCM accounts of %s to '%s'", str(self), str(accounts_list)) + api = User.api_for(c.UPDATE, self.endpoint) if self.endpoint.version() >= (10, 4, 0): - r = self.patch(f"{User.api_for(c.UPDATE, self.endpoint)}/{self.id}", params={"scmAccounts": accounts_list}) + r = self.patch(f"{api}/{self.id}", params={"scmAccounts": accounts_list}) else: params = self.api_params() params["scmAccount"] = ",".join(set(accounts_list)) - r = self.post(User.api_for(c.UPDATE, self.endpoint), params=params) + r = self.post(api, params=params) if not r.ok: self.scm_accounts = [] return False diff --git a/test/integration/it.sh b/test/integration/it.sh index 808944529..91cccb438 100755 --- a/test/integration/it.sh +++ b/test/integration/it.sh @@ -97,7 +97,7 @@ do logmsg "Creating IT test environment $env - sonarId $id" sqport=$IT_TEST_PORT pgport=$(expr $sqport - 4000) - # echo sonar create -i $id -t "$(tag_for "$env")" -s $sqport -p 6020 -f "$(backup_for "$env")" + echo sonar create -i $id -t "$(tag_for "$env")" -s $sqport -p $pgport -f "$(backup_for "$env")" sonar create -i $id -t "$(tag_for "$env")" -s $sqport -p $pgport -f "$(backup_for "$env")" 1>$IT_LOG_FILE 2>&1 export SONAR_TOKEN=$SONAR_TOKEN_ADMIN_USER export SONAR_HOST_URL="http://localhost:$sqport" diff --git a/test/unit/test_cli.py b/test/unit/test_cli.py index 4960bab61..9cd91d619 100644 --- a/test/unit/test_cli.py +++ b/test/unit/test_cli.py @@ -23,7 +23,7 @@ from collections.abc import Generator import json -import logging +from copy import deepcopy import pytest from sonar import exceptions, projects, utilities as sutil from cli import options as opt @@ -36,26 +36,25 @@ def test_import_compatibility() -> None: jsondata = util.SQ.basics() assert proj_cli.__check_sq_environments(util.SQ, jsondata) is None - incompatible_data = jsondata.copy() - incompatible_data["version"] = "8.7.0" + src_json = deepcopy(jsondata) + src_json["version"] = "8.7.0" with pytest.raises(exceptions.UnsupportedOperation): - proj_cli.__check_sq_environments(util.SQ, incompatible_data) + proj_cli.__check_sq_environments(util.SQ, src_json) - incompatible_data = jsondata.copy() - incompatible_data["plugins"] = {} - with pytest.raises(exceptions.UnsupportedOperation): - proj_cli.__check_sq_environments(util.SQ, incompatible_data) + src_json = deepcopy(jsondata) + src_json["plugins"] = {} + assert proj_cli.__check_sq_environments(util.SQ, src_json) is None - incompatible_data["plugins"] = jsondata["plugins"].copy() - incompatible_data["plugins"]["lua"] = "1.0 [Lua Analyzer]" + src_json = deepcopy(jsondata) + src_json["plugins"]["lua"] = "1.0 [Lua Analyzer]" with pytest.raises(exceptions.UnsupportedOperation): - proj_cli.__check_sq_environments(util.SQ, incompatible_data) + proj_cli.__check_sq_environments(util.SQ, src_json) - incompatible_data["plugins"] = jsondata["plugins"].copy() - for p, v in incompatible_data["plugins"].items(): - incompatible_data["plugins"][p] = "1." + v + src_json = deepcopy(jsondata) + for p, v in src_json["plugins"].items(): + src_json["plugins"][p] = "1." + v with pytest.raises(exceptions.UnsupportedOperation): - proj_cli.__check_sq_environments(util.SQ, incompatible_data) + proj_cli.__check_sq_environments(util.SQ, src_json) def test_import(get_json_file: Generator[str]) -> None: diff --git a/test/unit/test_findings.py b/test/unit/test_findings.py index 6ffe023f9..5b48072b6 100644 --- a/test/unit/test_findings.py +++ b/test/unit/test_findings.py @@ -370,9 +370,9 @@ def test_issues_count_3() -> None: def test_search_issues_by_project() -> None: """test_search_issues_by_project""" nb_issues = len(issues.search_by_project(endpoint=util.SQ, project_key=util.LIVE_PROJECT, search_findings=True)) - assert 200 <= nb_issues <= 1000 + assert 100 <= nb_issues <= 500 nb_issues = len(issues.search_by_project(endpoint=util.SQ, project_key=util.LIVE_PROJECT, params={"resolved": "false"})) - assert nb_issues < 1000 + assert nb_issues < 500 nb_issues = len(issues.search_by_project(endpoint=util.SQ, project_key=None)) assert nb_issues > 1000 diff --git a/test/unit/test_hotspots.py b/test/unit/test_hotspots.py new file mode 100644 index 000000000..202c65034 --- /dev/null +++ b/test/unit/test_hotspots.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +# +# sonar-tools tests +# Copyright (C) 2024-2025 Olivier Korach +# mailto:olivier.korach AT gmail DOT com +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3 of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# + +""" Test of the hotspots module and class, as well as changelog """ + +import utilities as tutil +from sonar import hotspots +from sonar.util import constants as c + + +def test_transitions() -> None: + hotspot_d = hotspots.search(endpoint=tutil.SQ, filters={"project": "okorach_sonar-tools"}) + hotspot = list(hotspot_d.values())[0] + + assert hotspot.mark_as_safe() + assert hotspot.reopen() + + assert hotspot.mark_as_to_review() + assert hotspot.reopen() diff --git a/test/unit/test_issues.py b/test/unit/test_issues.py index 9931ffa71..3746dfc4b 100644 --- a/test/unit/test_issues.py +++ b/test/unit/test_issues.py @@ -137,7 +137,8 @@ def test_changelog() -> None: assert str(issue) == f"Issue key '{issue_key}'" assert issue.is_false_positive() changelog_l = list(issue.changelog().values()) - assert len(changelog_l) == 1 + nb_changes = 3 if tutil.SQ.version() >= (2025, 1, 0) else 1 + assert len(changelog_l) == nb_changes changelog = changelog_l[0] assert changelog.is_resolve_as_fp() assert not changelog.is_closed() @@ -190,13 +191,12 @@ def test_transitions() -> None: assert issue.reopen() assert not issue.reopen() - assert issue.mark_as_wont_fix() - assert not issue.mark_as_wont_fix() - assert issue.reopen() - assert not issue.reopen() - - assert issue.accept() - assert not issue.accept() + if tutil.SQ.version() >= (10, 2, 0): + assert issue.accept() + assert not issue.accept() + else: + assert issue.mark_as_wont_fix() + assert not issue.mark_as_wont_fix() assert issue.reopen() assert not issue.reopen() diff --git a/test/unit/test_projects.py b/test/unit/test_projects.py index 1742b1955..9493e9c0d 100644 --- a/test/unit/test_projects.py +++ b/test/unit/test_projects.py @@ -202,12 +202,19 @@ def test_set_quality_gate(get_test_project: Generator[projects.Project], get_tes def test_ai_code_assurance(get_test_project: Generator[projects.Project]) -> None: """test_ai_code_assurance""" - if util.SQ.edition() == "community": - pytest.skip("AI Code Fix not available in SonarQube Community Build") - if util.SQ.version() >= (10, 7, 0): + proj = get_test_project + if util.SQ.version() >= (10, 7, 0) and util.SQ.edition() != "community": proj = get_test_project assert proj.set_contains_ai_code(True) - assert proj.get_ai_code_assurance() in ("CONTAINS_AI_CODE", "AI_CODE_ASSURED") + assert proj.get_ai_code_assurance() in ( + "CONTAINS_AI_CODE", + "AI_CODE_ASSURED", + "AI_CODE_ASSURANCE_ON", + "AI_CODE_ASSURANCE_OFF", + "AI_CODE_ASSURANCE_PASS", + "AI_CODE_ASSURANCE_FAIL", + "NONE", + ) assert proj.set_contains_ai_code(False) assert proj.get_ai_code_assurance() == "NONE" proj.key = util.NON_EXISTING_KEY @@ -215,6 +222,9 @@ def test_ai_code_assurance(get_test_project: Generator[projects.Project]) -> Non assert proj.get_ai_code_assurance() is None assert not proj.set_contains_ai_code(False) assert proj.get_ai_code_assurance() is None + else: + with pytest.raises(exceptions.UnsupportedOperation): + proj.get_ai_code_assurance() def test_set_quality_profile(get_test_project: Generator[projects.Project], get_test_qp: Generator[qualityprofiles.QualityProfile]) -> None: diff --git a/test/unit/test_qp.py b/test/unit/test_qp.py index 3c419c101..a542f654b 100644 --- a/test/unit/test_qp.py +++ b/test/unit/test_qp.py @@ -156,9 +156,9 @@ def test_add_remove_rules(get_test_qp: Generator[qualityprofiles.QualityProfile] def test_import() -> None: """test_import""" - util.start_logging("INFO") rules.get_list(util.TEST_SQ) - # delete all portfolios in test + # delete all quality profiles in test + _ = [qp.set_as_default() for qp in qualityprofiles.get_list(util.TEST_SQ).values() if qp.name == util.SONAR_WAY] qp_list = set(o for o in qualityprofiles.get_list(util.TEST_SQ, use_cache=False).values() if not o.is_built_in and not o.is_default) _ = [o.delete() for o in qp_list] with open("test/files/config.json", "r", encoding="utf-8") as f: @@ -167,9 +167,9 @@ def test_import() -> None: # Compare QP list json_name_list = sorted([k for k, v in qualityprofiles.flatten(json_exp).items() if not v.get("isBuiltIn", False)]) - qp_name_list = sorted([f"{o.language}:{o.name}" for o in qualityprofiles.get_list(util.TEST_SQ).values() if not o.is_built_in]) - logging.info("FLAT LIST2 = %s", str(json_name_list)) - logging.info("FLAT LIST2 = %s", str(qp_name_list)) + qp_name_list = sorted([f"{o.language}:{o.name}" for o in qualityprofiles.get_list(util.TEST_SQ, use_cache=False).values() if not o.is_built_in]) + logging.debug("Imported list = %s", str(json_name_list)) + logging.debug("SonarQube list = %s", str(qp_name_list)) assert json_name_list == qp_name_list diff --git a/test/unit/test_users.py b/test/unit/test_users.py index c3a4b048d..abe60da87 100644 --- a/test/unit/test_users.py +++ b/test/unit/test_users.py @@ -22,7 +22,7 @@ """ users tests """ from collections.abc import Generator - +from datetime import datetime import pytest import utilities as util @@ -183,16 +183,18 @@ def test_update(get_test_user: Generator[users.User]) -> None: assert sorted(user.scm_accounts) == sorted(["foo@gmail.com", "bar@gmail.com", "foo", "bar"]) if util.SQ.version() >= (10, 4, 0): - user.update(login="johndoe") - assert user.login == "johndoe" + new_login = f"johndoe{str(datetime.now()).replace(' ', '').replace(':', '')}" + user.update(login=new_login) + assert user.login == new_login user.update(name="John Doe", email="john@doe.com") assert user.name == "John Doe" assert user.email == "john@doe.com" if util.SQ.version() >= (10, 4, 0): - user.update(login="jdoe", email="john@doe.com") - assert user.login == "jdoe" + new_login = f"johndoe{str(datetime.now()).replace(' ', '-').replace(':', '-')}" + user.update(login=new_login, email="john@doe.com") + assert user.login == new_login def test_set_groups(get_test_user: Generator[users.User]) -> None: @@ -208,6 +210,7 @@ def test_set_groups(get_test_user: Generator[users.User]) -> None: def test_import() -> None: data = {} users.import_config(util.SQ, data) + now_str = {str(datetime.now()).replace(" ", "-").replace(":", "-")} data = { "users": { "TEMP": {"local": True, "name": "User name TEMP", "scmAccounts": "temp@acme.com, temp@gmail.com"}, @@ -216,7 +219,7 @@ def test_import() -> None: "groups": "sonar-administrators", "local": True, "name": "User name TEMP_ADMIN", - "scmAccounts": "admin-acme, administrator-acme", + "scmAccounts": f"admin-acme{now_str}, administrator-acme{now_str}", }, } } From 2d1ea2e1bf9a7654407a024a89257a7419b874eb Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sat, 8 Mar 2025 13:57:48 +0100 Subject: [PATCH 13/24] Clean-up-audit-history (#1582) * Remove unneeded code * add test on get versions * Fix bug on get_versions() * Test get_versions() --- sonar/components.py | 10 ++-------- test/unit/test_projects.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/sonar/components.py b/sonar/components.py index 236bd44d0..afd9242aa 100644 --- a/sonar/components.py +++ b/sonar/components.py @@ -261,11 +261,8 @@ def get_analyses(self, filter_in: Optional[list[str]] = None, filter_out: Option def get_versions(self) -> dict[str, datetime]: """Returns a dict of project versions and their dates""" - data = { - a["version"]: utilities.string_to_date(a["date"]) - for a in reversed(self.get_analyses(filter_in=["VERSION"], filter_out=["QUALITY_GATE", "QUALITY_PROFILE", "SQ_UPGRADE"])) - } - log.debug("Component versions = %s", utilities.json_dump(data)) + data = {a["projectVersion"]: utilities.string_to_date(a["date"]) for a in reversed(self.get_analyses(filter_in=["VERSION"]))} + log.debug("Component versions = %s", str(data.keys())) return data def get_ai_code_assurance(self) -> Optional[str]: @@ -311,9 +308,6 @@ def _audit_history_retention(self, audit_settings: types.ConfigSettings) -> list :param dict audit_settings: Options of what to audit and thresholds to raise problems :return: List of problems found, or empty list """ - if not audit_settings.get("audit.projects.historyRetention", True): - log.debug("%s: History retention audit disabled, audit skipped", str(self)) - return [] max_history = audit_settings.get("audit.projects.maxHistoryCount", 100) if max_history == 0: log.debug("Auditing %s history retention disabled, skipped...", str(self)) diff --git a/test/unit/test_projects.py b/test/unit/test_projects.py index 9493e9c0d..e8418e088 100644 --- a/test/unit/test_projects.py +++ b/test/unit/test_projects.py @@ -114,6 +114,19 @@ def test_webhooks() -> None: assert len(proj.webhooks()) == 0 +def test_versions() -> None: + """test_versions""" + proj = projects.Project.get_object(endpoint=util.SQ, key=util.LIVE_PROJECT) + vers = proj.get_versions() + v_list = list(vers.keys()) + assert len(v_list) > 10 + for v in "3.9", "2.11", "2.9", "v1.13": + assert v in v_list + if proj.endpoint.version() >= (2025, 1, 0): + for v in "3.1", "3.4", "3.7": + assert v in v_list + + def test_count() -> None: """test_count""" assert projects.count(util.SQ) > 30 From d6fbd9e1fd328f6aef81e7a1c0c3ab1028b0f61f Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sat, 8 Mar 2025 14:31:42 +0100 Subject: [PATCH 14/24] Upgrade alpine (#1583) --- conf/release.Dockerfile | 2 +- conf/snapshot.Dockerfile | 2 +- migration/release.Dockerfile | 2 +- migration/snapshot.Dockerfile | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conf/release.Dockerfile b/conf/release.Dockerfile index be25c6764..17d5597ae 100644 --- a/conf/release.Dockerfile +++ b/conf/release.Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.21.2 +FROM alpine:3.21.3 LABEL maintainer="olivier.korach@gmail.com" ENV IN_DOCKER="Yes" diff --git a/conf/snapshot.Dockerfile b/conf/snapshot.Dockerfile index 5b398eb3e..74762696c 100644 --- a/conf/snapshot.Dockerfile +++ b/conf/snapshot.Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.21.2 +FROM alpine:3.21.3 LABEL maintainer="olivier.korach@gmail.com" ENV IN_DOCKER="Yes" diff --git a/migration/release.Dockerfile b/migration/release.Dockerfile index 4c813756a..8becc3260 100644 --- a/migration/release.Dockerfile +++ b/migration/release.Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.21.2 +FROM alpine:3.21.3 LABEL maintainer="olivier.korach@gmail.com" ENV IN_DOCKER="Yes" diff --git a/migration/snapshot.Dockerfile b/migration/snapshot.Dockerfile index 7a83ba9d6..a6966efbd 100644 --- a/migration/snapshot.Dockerfile +++ b/migration/snapshot.Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.21.2 +FROM alpine:3.21.3 LABEL maintainer="olivier.korach@gmail.com" ENV IN_DOCKER="Yes" From 6ad23a3a3cb01d4ae30e71f4ae36b2dc143f34d3 Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sat, 8 Mar 2025 15:28:49 +0100 Subject: [PATCH 15/24] Audit accepted and FP issues (#1584) * Update doc * Add default settings * Add problems for too many FP or accepted issues * Fix settings * Separate FP and accepted thresholds * Add audit of accepted and FP issues * Add audit of FP and accepted for projects and branches * Fix measures extraction * Fixes * Reduce threahold to 1 Accepted or FP per 500 LOC --- doc/audit-settings.md | 20 ++++++++++++++++++-- doc/sonar-audit.md | 2 ++ sonar/audit/rules.json | 12 ++++++++++++ sonar/audit/rules.py | 2 ++ sonar/audit/sonar-audit.properties | 11 +++++++++-- sonar/branches.py | 1 + sonar/components.py | 21 +++++++++++++++++++++ sonar/projects.py | 1 + 8 files changed, 66 insertions(+), 4 deletions(-) diff --git a/doc/audit-settings.md b/doc/audit-settings.md index 5a11cd45d..e576a3e89 100644 --- a/doc/audit-settings.md +++ b/doc/audit-settings.md @@ -134,11 +134,12 @@ audit.projects.permissions.maxHotspotAdminGroups = 2 #========================= PROJECT AUDIT CONFIGURATION ======================== -# Audit and warn) for projects likely to be duplicates + +# Audit (and warn) for projects likely to be duplicates # Duplicate projects are detected from project keys that are similar audit.projects.duplicates = yes -# Audit and warn) for projects that have been provisioned but never analyzed +# Audit (and warn) for projects that have been provisioned but never analyzed audit.projects.neverAnalyzed = yes # Audit (and warn) if project visibility is public @@ -159,6 +160,18 @@ audit.projects.suspiciousExclusionsExceptions = \\*\\*/(__pycache__|libs|lib|ven # Set property to 0 to turn off the check audit.projects.maxLastAnalysisAge = 180 +# Audit (and warn) for projects/project branches with too many data points in history +# Set property to 0 to turn off the check +audit.projects.maxHistoryCount = 100 + +# Define project minimum size to audit accepted/FP issues +audit.projects.minLocSize = 10000 +# Audit (and warn) for projects/project branches with too many accepted or false positive issues +# Set parameter to the minimum nbr of LoC per accepted or FP issues +# Set property to 0 to turn off the check +audit.projects.minLocPerAcceptedIssue = 500 +audit.projects.minLocPerFalsePositiveIssue = 500 + # Audit branches for zero LoC and last analysis date audit.projects.branches = yes @@ -194,6 +207,9 @@ audit.projects.failedTasks = yes # Audits projects analyzed with too old scanner version, 2 years by default audit.projects.scannerMaxAge = 730 +# Audits projects scanner consistency with project type +audit.projects.scanner = yes + #====================== QUALITY GATES AUDIT CONFIGURATION ===================== # Audit that there are not too many quality gates, this defeats company common governance diff --git a/doc/sonar-audit.md b/doc/sonar-audit.md index 1e0081796..786dc503d 100644 --- a/doc/sonar-audit.md +++ b/doc/sonar-audit.md @@ -196,6 +196,8 @@ sonar-audit --what projects -f projectsAudit.csv --csvSeparator ';' - Projects analyzed with apparently a wrong scanner (Can't be certain in all cases) - Projects with too many analysis history data points (due to wrong housekeeping settings or wrong usage of `sonar.projectVersion`) + - Projects with too many accepted or false positive issues (Projects > 10K LoC and more than 1 accepted/FP issue + per 500 LoC) - Branches: (if `audit.project.branches = yes`, default `yes`) - Branches never analyzed but marked as "keep when inactive" - Portfolios: (if `audit.applications = yes`, default `yes`) diff --git a/sonar/audit/rules.json b/sonar/audit/rules.json index db2c4f20d..371e17d5c 100644 --- a/sonar/audit/rules.json +++ b/sonar/audit/rules.json @@ -296,6 +296,18 @@ "object": "Branch", "message": "{} has {} data points in history, this is excessive and causes excessive DB size" }, + "PROJ_TOO_MANY_ACCEPTED": { + "severity": "MEDIUM", + "type": "GOVERNANCE", + "object": "Branch", + "message": "{} has {} accepted issues for {} LoC, this is excessive" + }, + "PROJ_TOO_MANY_FP": { + "severity": "MEDIUM", + "type": "GOVERNANCE", + "object": "Branch", + "message": "{} has {} false positive issues for {} LoC, this is excessive" + }, "BRANCH_NEVER_ANALYZED": { "severity": "LOW", "type": "PERFORMANCE", diff --git a/sonar/audit/rules.py b/sonar/audit/rules.py index 687497213..af2ea256f 100644 --- a/sonar/audit/rules.py +++ b/sonar/audit/rules.py @@ -135,6 +135,8 @@ class RuleId(enum.Enum): PROJ_SCM_UNDETECTED = 1312 PROJ_WRONG_SCANNER = 1313 PROJ_HISTORY_COUNT = 1314 + PROJ_TOO_MANY_ACCEPTED = 1315 + PROJ_TOO_MANY_FP = 1316 NOT_USING_BRANCH_ANALYSIS = 1400 SIF_UNDETECTED_SCM = 1401 diff --git a/sonar/audit/sonar-audit.properties b/sonar/audit/sonar-audit.properties index 9ea9db1fc..39b4be578 100644 --- a/sonar/audit/sonar-audit.properties +++ b/sonar/audit/sonar-audit.properties @@ -148,11 +148,11 @@ audit.projects.permissions.maxHotspotAdminGroups = 2 #========================= PROJECT AUDIT CONFIGURATION ======================== -# Audit and warn) for projects likely to be duplicates +# Audit (and warn) for projects likely to be duplicates # Duplicate projects are detected from project keys that are similar audit.projects.duplicates = yes -# Audit and warn) for projects that have been provisioned but never analyzed +# Audit (and warn) for projects that have been provisioned but never analyzed audit.projects.neverAnalyzed = yes # Audit (and warn) if project visibility is public @@ -177,6 +177,13 @@ audit.projects.maxLastAnalysisAge = 180 # Set property to 0 to turn off the check audit.projects.maxHistoryCount = 100 +# Define project minimum size to audit accepted/FP issues +audit.projects.minLocSize = 10000 +# Audit (and warn) for projects/project branches with too many accepted or false positive issues +# Set parameter to the minimum nbr of LoC per accepted or FP issues +# Set property to 0 to turn off the check +audit.projects.minLocPerAcceptedIssue = 500 +audit.projects.minLocPerFalsePositiveIssue = 500 # Audit branches for zero LoC and last analysis date audit.projects.branches = yes diff --git a/sonar/branches.py b/sonar/branches.py index 3f0b8216f..92cdb3ce6 100644 --- a/sonar/branches.py +++ b/sonar/branches.py @@ -375,6 +375,7 @@ def audit(self, audit_settings: types.ConfigSettings) -> list[Problem]: + self.__audit_never_analyzed() + self._audit_bg_task(audit_settings) + self._audit_history_retention(audit_settings) + + self._audit_accepted_or_fp_issues(audit_settings) ) except Exception as e: log.error("%s while auditing %s, audit skipped", util.error_msg(e), str(self)) diff --git a/sonar/components.py b/sonar/components.py index afd9242aa..5e7b8b4ad 100644 --- a/sonar/components.py +++ b/sonar/components.py @@ -322,6 +322,27 @@ def _audit_history_retention(self, audit_settings: types.ConfigSettings) -> list return [Problem(get_rule(RuleId.PROJ_HISTORY_COUNT), self, str(self), history_len)] return [] + def _audit_accepted_or_fp_issues(self, audit_settings: types.ConfigSettings) -> list[Problem]: + """Audits whether a project or branch has too many accepted or FP issues + + :param dict audit_settings: Options of what to audit and thresholds to raise problems + :return: List of problems found, or empty list + """ + loc_min = audit_settings.get("audit.projects.minLocSize", 10000) + fp_min = audit_settings.get("minLocPerFalsePositiveIssue", 500) + accepted_min = audit_settings.get("minLocPerAcceptedIssue", 500) + m_list = ["ncloc", "accepted_issues", "false_positive_issues"] + d = {k: int(v.value) if v is not None else 0 for k, v in self.get_measures(m_list).items()} + ncloc, nb_accepted, nb_fp = d["ncloc"], d["accepted_issues"], d["false_positive_issues"] + problems = [] + if ncloc < loc_min: + return problems + if nb_accepted * accepted_min > ncloc: + problems.append(Problem(get_rule(RuleId.PROJ_TOO_MANY_ACCEPTED), self, str(self), nb_accepted, ncloc)) + if nb_fp * fp_min > ncloc: + problems.append(Problem(get_rule(RuleId.PROJ_TOO_MANY_FP), self, str(self), nb_fp, ncloc)) + return problems + def last_task(self) -> Optional[tasks.Task]: """Returns the last analysis background task of a problem, or none if not found""" return tasks.search_last(component_key=self.key, endpoint=self.endpoint) diff --git a/sonar/projects.py b/sonar/projects.py index dff755b68..f90441d05 100644 --- a/sonar/projects.py +++ b/sonar/projects.py @@ -685,6 +685,7 @@ def audit(self, audit_settings: types.ConfigSettings, write_q: Queue[list[Proble problems += self.__audit_binding_valid(audit_settings) problems += self.__audit_scanner(audit_settings) problems += self._audit_history_retention(audit_settings) + problems += self._audit_accepted_or_fp_issues(audit_settings) except (ConnectionError, RequestException) as e: util.handle_error(e, f"auditing {str(self)}", catch_all=True) From c389ccca14048d88c5df0cb853533758113a31bc Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sat, 8 Mar 2025 15:36:18 +0100 Subject: [PATCH 16/24] Fixes #1579 (#1585) --- sonar/settings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sonar/settings.py b/sonar/settings.py index 75779a2f7..78aac07ac 100644 --- a/sonar/settings.py +++ b/sonar/settings.py @@ -322,7 +322,7 @@ def is_settable(self) -> bool: def category(self) -> tuple[str, str]: """Returns the 2 levels classification of a setting""" m = re.match( - r"^sonar\.(cpd\.)?(abap|androidLint|apex|azureresourcemanager|cloudformation|c|cpp|cfamily|cobol|cs|css|dart|docker|" + r"^sonar\.(cpd\.)?(abap|androidLint|ansible|apex|azureresourcemanager|cloudformation|c|cpp|cfamily|cobol|cs|css|dart|docker|" r"eslint|flex|go|html|java|javascript|jcl|json|jsp|kotlin|objc|php|pli|plsql|python|ipynb|rpg|ruby|scala|swift|" r"terraform|text|tsql|typescript|vb|vbnet|xml|yaml)\.", self.key, @@ -359,6 +359,7 @@ def category(self) -> tuple[str, str]: r"sonar\.cpd|sonar\.dbcleaner|sonar\.developerAggregatedInfo|sonar\.governance|sonar\.issues|sonar\.lf|sonar\.notifications|" r"sonar\.portfolios|sonar\.qualitygate|sonar\.scm\.disabled|sonar\.scm\.provider|sonar\.technicalDebt|sonar\.validateWebhooks|" r"sonar\.docker|sonar\.login|sonar\.kubernetes|sonar\.plugins|sonar\.documentation|sonar\.projectCreation|" + r"sonar\.autodetect\.ai\.code|sonar\.pdf\.confidential\.header\.enabled|sonar\.scanner\.skipNodeProvisioning|" r"sonar\.qualityProfiles|sonar\.announcement|provisioning\.git|sonar\.ce|sonar\.azureresourcemanager|sonar\.filesize\.limit).*$", self.key, ): From e814ebbd24e83031bdf32fb1718acb5c397b5d28 Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sun, 9 Mar 2025 13:31:03 +0100 Subject: [PATCH 17/24] Replace Accepted by WF on 9.9 (#1586) * Replace Accepted by WF on 9.9 * Fix error on AI Code assurance on 9.9 * Delete empty external issues files before scan * Fix missing RED color * Delete environment on tear down * Fix LTS token --- conf/run_linters.sh | 2 ++ sonar/components.py | 10 +++++++--- test/integration/it-tools.sh | 1 + test/integration/it.sh | 21 +++++++++++++++------ 4 files changed, 25 insertions(+), 9 deletions(-) diff --git a/conf/run_linters.sh b/conf/run_linters.sh index f703dccf6..a8a6c391b 100755 --- a/conf/run_linters.sh +++ b/conf/run_linters.sh @@ -51,6 +51,7 @@ flake8 --config "$CONFIG/.flake8" "$ROOTDIR" >"$flake8Report" if [ "$localbuild" = "true" ]; then echo "===> Running shellcheck" shellcheck "$ROOTDIR"/*.sh "$ROOTDIR"/*/*.sh -s bash -f json | "$CONFDIR"/shellcheck2sonar.py >"$shellcheckReport" + [ ! -s "$shellcheckReport" ] && rm -f "$shellcheckReport" echo "===> Running checkov" checkov -d . --framework dockerfile -o sarif --output-file-path "$buildDir" @@ -59,4 +60,5 @@ if [ "$localbuild" = "true" ]; then "$CONFDIR"/build.sh docker trivy image -f json -o "$buildDir"/trivy_results.json olivierkorach/sonar-tools:latest python3 "$CONFDIR"/trivy2sonar.py < "$buildDir"/trivy_results.json > "$trivyReport" + [ ! -s "$trivyReport" ] && rm -f "$trivyReport" fi \ No newline at end of file diff --git a/sonar/components.py b/sonar/components.py index 5e7b8b4ad..a79cbcea6 100644 --- a/sonar/components.py +++ b/sonar/components.py @@ -277,7 +277,7 @@ def get_ai_code_assurance(self) -> Optional[str]: return str(json.loads(self.get(api, params=self.api_params(c.GET)).text)["aiCodeAssurance"]).upper() except (ConnectionError, RequestException) as e: utilities.handle_error(e, f"getting AI code assurance of {str(self)}", catch_all=True) - if "Unknown url" in str(e): + if "Unknown url" in utilities.error_msg(e): raise exceptions.UnsupportedOperation( f"AI code assurance is not available for {self.endpoint.edition()} edition version {str(self.endpoint.version())}" ) @@ -331,9 +331,13 @@ def _audit_accepted_or_fp_issues(self, audit_settings: types.ConfigSettings) -> loc_min = audit_settings.get("audit.projects.minLocSize", 10000) fp_min = audit_settings.get("minLocPerFalsePositiveIssue", 500) accepted_min = audit_settings.get("minLocPerAcceptedIssue", 500) - m_list = ["ncloc", "accepted_issues", "false_positive_issues"] + m_list = ["ncloc", "false_positive_issues", "accepted_issues" if self.endpoint.version() >= (10, 2, 0) else "wont_fix_issues"] d = {k: int(v.value) if v is not None else 0 for k, v in self.get_measures(m_list).items()} - ncloc, nb_accepted, nb_fp = d["ncloc"], d["accepted_issues"], d["false_positive_issues"] + ncloc, nb_accepted, nb_fp = ( + d["ncloc"], + d["accepted_issues" if self.endpoint.version() >= (10, 2, 0) else "wont_fix_issues"], + d["false_positive_issues"], + ) problems = [] if ncloc < loc_min: return problems diff --git a/test/integration/it-tools.sh b/test/integration/it-tools.sh index 83f78e5d7..4084a1b92 100644 --- a/test/integration/it-tools.sh +++ b/test/integration/it-tools.sh @@ -27,6 +27,7 @@ IT_LOG_FILE="$TMP/it.log" mkdir -p "$TMP" YELLOW=$(tput setaf 3) +RED=$(tput setaf 1) GREEN=$(tput setaf 2) RESET=$(tput setaf 7) diff --git a/test/integration/it.sh b/test/integration/it.sh index 91cccb438..84178defa 100755 --- a/test/integration/it.sh +++ b/test/integration/it.sh @@ -100,6 +100,10 @@ do echo sonar create -i $id -t "$(tag_for "$env")" -s $sqport -p $pgport -f "$(backup_for "$env")" sonar create -i $id -t "$(tag_for "$env")" -s $sqport -p $pgport -f "$(backup_for "$env")" 1>$IT_LOG_FILE 2>&1 export SONAR_TOKEN=$SONAR_TOKEN_ADMIN_USER + if [[ "$env" =~ ^lts.*$ ]]; then + logmsg "Using LTS token" + export SONAR_TOKEN=$SONAR_TOKEN_LTS_ADMIN_USER + fi export SONAR_HOST_URL="http://localhost:$sqport" fi @@ -176,10 +180,10 @@ do logmsg "sonar-projects-export $env SKIPPED" else logmsg "=====> IT sonar-findings-export $env USER export" - if [ "$env" = "lts" ]; then - export SONAR_TOKEN=$SONAR_TOKEN_USER_USER_LTS - else - export SONAR_TOKEN=$SONAR_TOKEN_USER_USER + export SONAR_TOKEN=$SONAR_TOKEN_USER_USER + if [[ "$env" =~ ^lts.*$ ]]; then + logmsg "Using LTS token" + export SONAR_TOKEN=$SONAR_TOKEN_LTS_USER_USER fi f2="findings-$env-user.csv"; run_test "$f2" sonar-findings-export -v DEBUG -k okorach_audio-video-tools,okorach_sonar-tools fi @@ -188,7 +192,12 @@ do logmsg "Restore sonar-tools last released version" pip install --force-reinstall sonar-tools 1>$IT_LOG_FILE 2>&1; - export SONAR_TOKEN="$SONAR_TOKEN_ADMIN_USER" + + export SONAR_TOKEN=$SONAR_TOKEN_ADMIN_USER + if [[ "$env" =~ ^lts.*$ ]]; then + logmsg "Using LTS token" + export SONAR_TOKEN=$SONAR_TOKEN_LTS_ADMIN_USER + fi logmsg "=====> IT released tools $env" f="measures-$env-rel.csv"; run_test "$f" sonar-measures-export -b -m _main --withURL f="findings-$env-rel.csv"; run_test "$f" sonar-findings-export @@ -217,7 +226,7 @@ do if [ "$env" != "sonarcloud" ]; then logmsg "Deleting environment sonarId $id" - # sonar delete -i "$id" 1>$IT_LOG_FILE 2>&1 + sonar delete -i "$id" 1>$IT_LOG_FILE 2>&1 fi done From 6bb43d9861626479f2f16274d055c9ddef76beae Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sun, 9 Mar 2025 13:51:58 +0100 Subject: [PATCH 18/24] Update 3.9 (#1587) --- doc/what-is-new.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/what-is-new.md b/doc/what-is-new.md index fdb8571a4..8be131e7c 100644 --- a/doc/what-is-new.md +++ b/doc/what-is-new.md @@ -1,5 +1,11 @@ # Version 3.9 +* Compatibility with SonarQube 2025.1 release +* Fixed sonar-projects import pre-check to be less strict (follow new SonarQube criterias for project import) +* A few new things audited by sonar-audit (Excessive project history data points and Excessive proportion of accepted or FP issues) +* Bug fixes +* More unit tests + # Version 3.8 * Bug fixes From ac248cc23fe4189cd453ee3c467c260cdfeca0c9 Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Fri, 21 Mar 2025 14:22:12 +0100 Subject: [PATCH 19/24] Bump to 3.10 (#1588) --- sonar/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sonar/version.py b/sonar/version.py index 12a234d8a..e3a0c06fb 100644 --- a/sonar/version.py +++ b/sonar/version.py @@ -24,5 +24,5 @@ """ -PACKAGE_VERSION = "3.9" +PACKAGE_VERSION = "3.10" MIGRATION_TOOL_VERSION = "0.5" From cd3b3b6b00e55aec4ebfc67c1c485fbcdd496579 Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Fri, 21 Mar 2025 14:38:01 +0100 Subject: [PATCH 20/24] Proper check on JRE version (#1591) * Fixes #1589 * Fixes #1592 --- sonar/sif_node.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sonar/sif_node.py b/sonar/sif_node.py index fdabb1255..a0e542ec4 100644 --- a/sonar/sif_node.py +++ b/sonar/sif_node.py @@ -40,7 +40,8 @@ _CE_TASKS = "Compute Engine Tasks" _WORKER_COUNT = "Worker Count" -JAVA_COMPAT = {17: [(9, 6, 0), (12, 0, 0)], 11: [(7, 9, 0), (9, 8, 0)], 8: [(7, 9, 0), (8, 9, 0)]} +_LATEST_SQS_VERSION = (9999, 99, 99) +JAVA_COMPAT = {21: [(25, 1, 0), _LATEST_SQS_VERSION], 17: [(9, 6, 0), _LATEST_SQS_VERSION], 11: [(7, 9, 0), (9, 8, 0)], 8: [(7, 9, 0), (8, 9, 0)]} def __audit_background_tasks(obj: object, obj_name: str) -> list[Problem]: From 0875e20f17284b1c0cecfbedef65a5da06a8529f Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Fri, 21 Mar 2025 18:22:58 +0100 Subject: [PATCH 21/24] Move config data out of code (#1593) * Add JVM compat as config data * Add config data in package * Read config data at module init * Check config data for JVM * Add scannerVersions * Add loading of scanner versions * Use config data for scanner versions * Fix scanners versions upload * Add recent scanners versions --- .gitignore | 1 + setup.py | 2 +- sonar/audit/__init__.py | 3 +- sonar/audit/config.json | 167 ++++++++++++++++++++++++++++++++++++++++ sonar/audit/config.py | 23 ++++++ sonar/sif_node.py | 10 +-- sonar/tasks.py | 123 +---------------------------- 7 files changed, 201 insertions(+), 128 deletions(-) create mode 100644 sonar/audit/config.json diff --git a/.gitignore b/.gitignore index 76d8d5f6d..0434eb4e0 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ tmp/ .venv/ .DS_Store +!sonar/audit/config.json !test/config.json !test/integration/sif*.json !.vscode/*.json diff --git a/setup.py b/setup.py index d583b6024..510a2cd9a 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ "Source Code": "https://github.com/okorach/sonar-tools", }, packages=setuptools.find_packages(), - package_data={"sonar": ["LICENSE", "audit/rules.json", "audit/sonar-audit.properties"]}, + package_data={"sonar": ["LICENSE", "audit/rules.json", "audit/config.json", "audit/sonar-audit.properties"]}, install_requires=[ "argparse", "datetime", diff --git a/sonar/audit/__init__.py b/sonar/audit/__init__.py index 447dc7f90..67d90ceb9 100644 --- a/sonar/audit/__init__.py +++ b/sonar/audit/__init__.py @@ -20,9 +20,10 @@ """sonar.audit module""" -from sonar.audit import rules +from sonar.audit import rules, config from sonar import utilities, errcodes +config.load_config_data() try: rules.load() except rules.RuleConfigError as e: diff --git a/sonar/audit/config.json b/sonar/audit/config.json new file mode 100644 index 000000000..70f3338ac --- /dev/null +++ b/sonar/audit/config.json @@ -0,0 +1,167 @@ +{ + "javaCompatibility": { + "21": [[2025, 1, 0], [9999, 99, 99]], + "17": [[9, 6, 0], [9999, 99, 99]], + "11": [[7, 9, 0], [9, 8, 0]], + "8": [[7, 9, 0], [8, 9, 0]] + }, + "scannerVersions": { + "ScannerCLI": { + "7.0.2": [2025, 2, 14], + "7.0.1": [2025, 2, 3], + "7.0.0": [2025, 1, 20], + "6.2.1": [2024, 10, 1], + "6.2.0": [2024, 9, 17], + "6.1.0": [2024, 6, 27], + "6.0.0": [2024, 6, 4], + "5.0.1": [2023, 8, 4], + "5.0.0": [2023, 7, 31], + "4.8.1": [2023, 8, 14], + "4.8.0": [2022, 2, 6], + "4.7.0": [2022, 2, 2], + "4.6.2": [2021, 5, 7], + "4.6.1": [2021, 4, 30], + "4.6.0": [2021, 1, 13], + "4.5.0": [2020, 10, 5], + "4.4.0": [2020, 7, 3], + "4.3.0": [2019, 3, 9], + "4.2.0": [2019, 10, 1], + "4.1.0": [2019, 9, 9] + }, + "ScannerMaven": { + "5.0.0": [2024, 11, 6], + "4.0.0": [2024, 5, 31], + "3.11.0": [2024, 3, 13], + "3.10.0": [2023, 9, 15], + "3.9.1": [2021, 12, 1], + "3.9.0": [2021, 4, 1], + "3.8.0": [2021, 1, 1], + "3.7.0": [2019, 10, 1], + "3.6.1": [2019, 8, 1], + "3.6.0": [2019, 1, 1], + "3.5.0": [2018, 9, 1], + "3.4.1": [2018, 6, 1], + "3.4.0": [2017, 11, 1], + "3.3.0": [2017, 3, 1], + "3.2.0": [2016, 9, 1], + "3.1.1": [2016, 9, 1], + "3.1.0": [2016, 9, 1], + "3.0.2": [2016, 4, 1], + "3.0.1": [2016, 1, 1], + "3.0.0": [2016, 1, 1] + }, + "ScannerGradle": { + "6.0.1": [2024, 11, 27], + "6.0.0": [2024, 11, 19], + "5.1.0": [2024, 7, 4], + "5.0.0": [2024, 3, 26], + "4.4.1": [2023, 10, 3], + "4.3.1": [2023, 9, 1], + "4.2.1": [2023, 6, 12], + "4.0.0": [2023, 2, 17], + "3.5.0": [2022, 10, 27], + "3.4.0": [2022, 6, 8], + "3.3.0": [2021, 6, 10], + "3.2.0": [2021, 4, 30], + "3.1.1": [2021, 1, 25], + "3.1.0": [2021, 1, 13], + "3.0.0": [2020, 6, 20], + "2.8.0": [2019, 10, 1] + }, + "ScannerMSBuild": { + "10.1.0": [2025, 3, 19], + "10.0.0": [2025, 3, 13], + "9.2.1": [2025, 2, 25], + "9.2.0": [2025, 2, 19], + "9.1.0": [2025, 2, 6], + "9.0.2": [2024, 11, 12], + "9.0.1": [2024, 10, 25], + "9.0.0": [2024, 9, 27], + "8.0.3": [2024, 9, 13], + "8.0.2": [2024, 9, 2], + "8.0.1": [2024, 8, 21], + "8.0.0": [2024, 8, 12], + "7.1.1": [2024, 7, 24], + "7.1.0": [2024, 7, 19], + "7.0.0": [2024, 7, 18], + "6.2.0": [2024, 2, 16], + "6.1.0": [2024, 1, 29], + "6.0.0": [2023, 12, 4], + "5.15.1": [2024, 3, 26], + "5.15.0": [2023, 11, 20], + "5.14.0": [2023, 10, 2], + "5.13.1": [2023, 8, 14], + "5.13.0": [2023, 4, 5], + "5.12.0": [2023, 3, 17], + "5.11.0": [2023, 1, 27], + "5.10.0": [2023, 1, 13], + "5.9.2": [2022, 12, 14], + "5.9.1": [2022, 12, 6], + "5.9.0": [2022, 12, 1], + "5.8.0": [2022, 8, 24], + "5.7.2": [2022, 7, 12], + "5.7.1": [2022, 6, 21], + "5.7.0": [2022, 6, 20], + "5.6.0": [2022, 5, 30], + "5.5.3": [2022, 2, 14], + "5.5.2": [2022, 2, 10], + "5.5.1": [2022, 2, 8], + "5.5.0": [2022, 2, 7], + "5.4.1": [2021, 12, 23], + "5.4.0": [2021, 11, 26], + "5.3.2": [2021, 10, 28], + "5.3.1": [2021, 9, 1], + "5.2.2": [2021, 6, 24], + "5.2.1": [2021, 4, 30], + "5.2.0": [2021, 4, 9], + "5.1.0": [2021, 3, 9], + "5.0.4": [2020, 11, 11], + "5.0.3": [2020, 11, 10], + "5.0.0": [2020, 11, 5], + "4.10.0": [2020, 6, 29], + "4.9.0": [2020, 5, 5], + "4.8.0": [2019, 11, 6], + "4.7.1": [2019, 9, 10], + "4.7.0": [2019, 9, 3] + }, + "ScannerNpm": { + "4.2.8": [2024, 3, 1], + "4.2.7": [2024, 2, 19], + "4.2.6": [2024, 12, 1], + "4.2.5": [2024, 10, 2], + "4.2.4": [2024, 10, 1], + "4.2.3": [2024, 8, 1], + "4.2.2": [2024, 7, 3], + "4.2.1": [2024, 7, 2], + "4.2.0": [2024, 7, 1], + "4.0.1": [2024, 6, 2], + "4.0.0": [2024, 6, 1], + "3.5.0": [2024, 5, 1], + "3.4.0": [2024, 3, 27], + "3.3.0": [2023, 11, 16], + "3.2.0": [2023, 11, 6], + "3.1.0": [2023, 8, 31], + "3.0.1": [2023, 2, 10], + "3.0.0": [2023, 1, 4], + "2.9.0": [2022, 12, 4], + "2.8.9": [2022, 12, 4], + "2.8.2": [2022, 9, 25], + "2.8.1": [2021, 6, 10], + "2.8.0": [2020, 11, 10], + "2.7.0": [2020, 7, 6], + "2.6.0": [2020, 3, 24], + "2.5.0": [2019, 6, 26], + "2.4.1": [2019, 5, 28], + "2.4.0": [2019, 2, 23], + "2.3.0": [2019, 2, 19], + "2.2.0": [2019, 2, 12], + "2.1.2": [2018, 10, 8], + "2.1.1": [2018, 8, 28], + "2.1.0": [2018, 7, 10] + }, + "Ant": { + "2.7.1": [2021, 4, 30], + "2.7": [2019, 10, 1] + } + } +} \ No newline at end of file diff --git a/sonar/audit/config.py b/sonar/audit/config.py index 7cb2de688..bab30d710 100644 --- a/sonar/audit/config.py +++ b/sonar/audit/config.py @@ -21,6 +21,8 @@ """sonar-config CLI""" import os import pathlib +import datetime +import json import jprops from typing import Optional import sonar.logging as log @@ -29,6 +31,8 @@ _CONFIG_SETTINGS = None +_CONFIG_DATA = None + def _load_properties_file(file: str) -> types.ConfigSettings: """Loads a properties file""" @@ -93,3 +97,22 @@ def configure() -> None: log.info("Creating file '%s'", config_file) with open(config_file, "w", encoding="utf-8") as fh: print(text, file=fh) + + +def load_config_data() -> None: + global _CONFIG_DATA + config_data_file = pathlib.Path(__file__).parent / "config.json" + with open(config_data_file, "r", encoding="utf-8") as fh: + text = fh.read() + _CONFIG_DATA = json.loads(text) + + +def get_java_compatibility() -> dict[int, list[tuple[int, int, int]]]: + return {int(k): [tuple(v[0]), tuple(v[1])] for k, v in _CONFIG_DATA["javaCompatibility"].items()} + + +def get_scanners_versions() -> dict[int, list[tuple[int, int, int]]]: + data = {} + for scanner, release_info in _CONFIG_DATA["scannerVersions"].items(): + data[scanner] = {k: datetime.datetime(v[0], v[1], v[2]) for k, v in release_info.items()} + return data diff --git a/sonar/sif_node.py b/sonar/sif_node.py index a0e542ec4..8e161c9b2 100644 --- a/sonar/sif_node.py +++ b/sonar/sif_node.py @@ -31,6 +31,7 @@ from sonar.util import types from sonar.audit.rules import get_rule, RuleId from sonar.audit.problem import Problem +import sonar.audit.config as audit_conf _RELEASE_DATE_6_7 = datetime.datetime(2017, 11, 8) + relativedelta(months=+6) _RELEASE_DATE_7_9 = datetime.datetime(2019, 7, 1) + relativedelta(months=+6) @@ -40,9 +41,6 @@ _CE_TASKS = "Compute Engine Tasks" _WORKER_COUNT = "Worker Count" -_LATEST_SQS_VERSION = (9999, 99, 99) -JAVA_COMPAT = {21: [(25, 1, 0), _LATEST_SQS_VERSION], 17: [(9, 6, 0), _LATEST_SQS_VERSION], 11: [(7, 9, 0), (9, 8, 0)], 8: [(7, 9, 0), (8, 9, 0)]} - def __audit_background_tasks(obj: object, obj_name: str) -> list[Problem]: """Audits the SIF for the health of background tasks stats, namely the failure rate @@ -149,10 +147,12 @@ def __audit_jvm_version(obj: object, obj_name: str, jvm_props: dict[str, str]) - except KeyError: log.warning("%s: Can't find SonarQube version in SIF, auditing this part is skipped", obj_name) return [] - if java_version not in JAVA_COMPAT: + java_compat = audit_conf.get_java_compatibility() + log.debug("Java compatibility matrix: %s", str(java_compat)) + if java_version not in java_compat: log.warning("%s: Java version %d not listed in compatibility matrix, skipping JVM version audit", obj_name, java_version) return [] - min_v, max_v = JAVA_COMPAT[java_version] + min_v, max_v = java_compat[java_version] sq_v_str = ".".join([str(i) for i in sq_version]) if min_v <= sq_version <= max_v: log.info("%s: SonarQube %s running on a supported java version (java %d)", obj_name, sq_v_str, java_version) diff --git a/sonar/tasks.py b/sonar/tasks.py index a9835a292..fc22083f2 100644 --- a/sonar/tasks.py +++ b/sonar/tasks.py @@ -34,6 +34,7 @@ import sonar.utilities as util from sonar.audit.rules import get_rule, RuleId from sonar.audit.problem import Problem +from sonar.audit.config import get_scanners_versions from sonar.util import types, cache SUCCESS = "SUCCESS" @@ -49,127 +50,7 @@ __SUSPICIOUS_EXCLUSIONS = None __SUSPICIOUS_EXCEPTIONS = None -SCANNER_VERSIONS = { - "ScannerCLI": { - "5.0.1": datetime.datetime(2023, 8, 4), - "5.0.0": datetime.datetime(2023, 7, 31), - "4.8.1": datetime.datetime(2023, 8, 14), - "4.8.0": datetime.datetime(2022, 2, 6), - "4.7.0": datetime.datetime(2022, 2, 2), - "4.6.2": datetime.datetime(2021, 5, 7), - "4.6.1": datetime.datetime(2021, 4, 30), - "4.6.0": datetime.datetime(2021, 1, 13), - "4.5.0": datetime.datetime(2020, 10, 5), - "4.4.0": datetime.datetime(2020, 7, 3), - "4.3.0": datetime.datetime(2019, 3, 9), - "4.2.0": datetime.datetime(2019, 10, 1), - "4.1.0": datetime.datetime(2019, 9, 9), - }, - "ScannerMaven": { - "3.11.0": datetime.datetime(2024, 3, 13), - "3.10.0": datetime.datetime(2023, 9, 15), - "3.9.1": datetime.datetime(2021, 12, 1), - "3.9.0": datetime.datetime(2021, 4, 1), - "3.8.0": datetime.datetime(2021, 1, 1), - "3.7.0": datetime.datetime(2019, 10, 1), - "3.6.1": datetime.datetime(2019, 8, 1), - "3.6.0": datetime.datetime(2019, 1, 1), - "3.5.0": datetime.datetime(2018, 9, 1), - "3.4.1": datetime.datetime(2018, 6, 1), - "3.4.0": datetime.datetime(2017, 11, 1), - "3.3.0": datetime.datetime(2017, 3, 1), - "3.2.0": datetime.datetime(2016, 9, 1), - "3.1.1": datetime.datetime(2016, 9, 1), - "3.1.0": datetime.datetime(2016, 9, 1), - "3.0.2": datetime.datetime(2016, 4, 1), - "3.0.1": datetime.datetime(2016, 1, 1), - "3.0.0": datetime.datetime(2016, 1, 1), - }, - "ScannerGradle": { - "5.0.0": datetime.datetime(2024, 3, 26), - "4.4.1": datetime.datetime(2023, 10, 3), - "4.3.1": datetime.datetime(2023, 9, 1), - "4.2.1": datetime.datetime(2023, 6, 12), - "4.0.0": datetime.datetime(2023, 2, 17), - "3.5.0": datetime.datetime(2022, 10, 27), - "3.4.0": datetime.datetime(2022, 6, 8), - "3.3.0": datetime.datetime(2021, 6, 10), - "3.2.0": datetime.datetime(2021, 4, 30), - "3.1.1": datetime.datetime(2021, 1, 25), - "3.1.0": datetime.datetime(2021, 1, 13), - "3.0.0": datetime.datetime(2020, 6, 20), - "2.8.0": datetime.datetime(2019, 10, 1), - }, - "ScannerMSBuild": { - "6.2.0": datetime.datetime(2024, 2, 16), - "6.1.0": datetime.datetime(2024, 1, 29), - "6.0.0": datetime.datetime(2023, 12, 4), - "5.15.1": datetime.datetime(2024, 3, 26), - "5.15.0": datetime.datetime(2023, 11, 20), - "5.14.0": datetime.datetime(2023, 10, 2), - "5.13.1": datetime.datetime(2023, 8, 14), - "5.13.0": datetime.datetime(2023, 4, 5), - "5.12.0": datetime.datetime(2023, 3, 17), - "5.11.0": datetime.datetime(2023, 1, 27), - "5.10.0": datetime.datetime(2023, 1, 13), - "5.9.2": datetime.datetime(2022, 12, 14), - "5.9.1": datetime.datetime(2022, 12, 6), - "5.9.0": datetime.datetime(2022, 12, 1), - "5.8.0": datetime.datetime(2022, 8, 24), - "5.7.2": datetime.datetime(2022, 7, 12), - "5.7.1": datetime.datetime(2022, 6, 21), - "5.7.0": datetime.datetime(2022, 6, 20), - "5.6.0": datetime.datetime(2022, 5, 30), - "5.5.3": datetime.datetime(2022, 2, 14), - "5.5.2": datetime.datetime(2022, 2, 10), - "5.5.1": datetime.datetime(2022, 2, 8), - "5.5.0": datetime.datetime(2022, 2, 7), - "5.4.1": datetime.datetime(2021, 12, 23), - "5.4.0": datetime.datetime(2021, 11, 26), - "5.3.2": datetime.datetime(2021, 10, 28), - "5.3.1": datetime.datetime(2021, 9, 1), - "5.2.2": datetime.datetime(2021, 6, 24), - "5.2.1": datetime.datetime(2021, 4, 30), - "5.2.0": datetime.datetime(2021, 4, 9), - "5.1.0": datetime.datetime(2021, 3, 9), - "5.0.4": datetime.datetime(2020, 11, 11), - "5.0.3": datetime.datetime(2020, 11, 10), - "5.0.0": datetime.datetime(2020, 11, 5), - "4.10.0": datetime.datetime(2020, 6, 29), - "4.9.0": datetime.datetime(2020, 5, 5), - "4.8.0": datetime.datetime(2019, 11, 6), - "4.7.1": datetime.datetime(2019, 9, 10), - "4.7.0": datetime.datetime(2019, 9, 3), - }, - "ScannerNpm": { - "3.5.0": datetime.datetime(2024, 5, 1), - "3.4.0": datetime.datetime(2024, 3, 27), - "3.3.0": datetime.datetime(2023, 11, 16), - "3.2.0": datetime.datetime(2023, 11, 6), - "3.1.0": datetime.datetime(2023, 8, 31), - "3.0.1": datetime.datetime(2023, 2, 10), - "3.0.0": datetime.datetime(2023, 1, 4), - "2.9.0": datetime.datetime(2022, 12, 4), - "2.8.9": datetime.datetime(2022, 12, 4), - "2.8.2": datetime.datetime(2022, 9, 25), - "2.8.1": datetime.datetime(2021, 6, 10), - "2.8.0": datetime.datetime(2020, 11, 10), - "2.7.0": datetime.datetime(2020, 7, 6), - "2.6.0": datetime.datetime(2020, 3, 24), - "2.5.0": datetime.datetime(2019, 6, 26), - "2.4.1": datetime.datetime(2019, 5, 28), - "2.4.0": datetime.datetime(2019, 2, 23), - "2.3.0": datetime.datetime(2019, 2, 19), - "2.2.0": datetime.datetime(2019, 2, 12), - "2.1.2": datetime.datetime(2018, 10, 8), - "2.1.1": datetime.datetime(2018, 8, 28), - "2.1.0": datetime.datetime(2018, 7, 10), - }, - "Ant": { - "2.7.1": datetime.datetime(2021, 4, 30), - "2.7": datetime.datetime(2019, 10, 1), - }, -} +SCANNER_VERSIONS = get_scanners_versions() class Task(sq.SqObject): From e496da97b517315a8a82c514a6f02a0331d2e939 Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Tue, 25 Mar 2025 12:45:29 +0100 Subject: [PATCH 22/24] Fix-issue-status-deprecation-in-search (#1600) * Remove unused symbols * Fixes #1595 --- sonar/findings.py | 6 ------ sonar/issues.py | 8 +++++++- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/sonar/findings.py b/sonar/findings.py index 40751d252..22205b352 100644 --- a/sonar/findings.py +++ b/sonar/findings.py @@ -96,10 +96,6 @@ "legacySeverity", ) -FILTERS = ("statuses", "resolutions", "severities", "languages", "pullRequest", "branch", "tags", "types", "createdBefore", "createdAfter") - -MQR_SEVERITIES = ("BLOCKER", "HIGH", "MEDIUM", "LOW", "INFO") -OLD_SEVERITIES = ("BLOCKER", "CRITICAL", "MAJOR", "LOW", "INFO") SEVERITY_NONE = "NONE" TYPE_VULN = "VULNERABILITY" @@ -108,12 +104,10 @@ TYPE_HOTSPOT = "SECURITY_HOTSPOT" TYPE_NONE = "NONE" -TYPES = (TYPE_VULN, TYPE_BUG, TYPE_CODE_SMELL, TYPE_HOTSPOT) QUALITY_SECURITY = "SECURITY" QUALITY_RELIABILITY = "RELIABILITY" QUALITY_MAINTAINABILITY = "MAINTAINABILITY" QUALITY_NONE = "NONE" -QUALITIES = (QUALITY_SECURITY, QUALITY_RELIABILITY, QUALITY_MAINTAINABILITY) # Mapping between old issues type and new software qualities TYPE_QUALITY_MAPPING = { diff --git a/sonar/issues.py b/sonar/issues.py index 24ba8b2de..dec2342a6 100644 --- a/sonar/issues.py +++ b/sonar/issues.py @@ -923,7 +923,13 @@ def pre_search_filters(endpoint: pf.Platform, params: ApiParams) -> ApiParams: if OLD_STATUS in filters: filters[OLD_STATUS] = util.list_re_value(filters[OLD_STATUS], mapping={NEW_FP: OLD_FP}) else: - # Starting from 10.4 - "resolutions" was renamed "issuesStatuses", "FALSE-POSITIVE" was renamed "FALSE_POSITIVE" + # Starting from 10.4 - "resolutions" and "statuses" are merged into "issuesStatuses", "FALSE-POSITIVE" was renamed "FALSE_POSITIVE" + # and "statuses" is deprecated + if "statuses" in filters: + if NEW_STATUS in filters: + filters[NEW_STATUS].update(filters.pop("statuses")) + else: + filters[NEW_STATUS] = filters.pop("statuses") filters = util.dict_remap(original_dict=filters, remapping={OLD_STATUS: NEW_STATUS}) if NEW_STATUS in filters: filters[NEW_STATUS] = util.list_re_value(filters[NEW_STATUS], mapping={OLD_FP: NEW_FP}) From ab7dd59b8b7245f33137cde89d738c28f19d2382 Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Tue, 25 Mar 2025 13:39:44 +0100 Subject: [PATCH 23/24] Fix-bug-on-permissions (#1601) * Remove unused symbols * Fixes #1595 * Fix bug on permissions * Change project keys * Fix no scanner context test --- sonar/permissions/permissions.py | 37 +++++++++--------------- sonar/permissions/project_permissions.py | 7 ++--- test/unit/test_apps.py | 10 +++---- test/unit/test_tasks.py | 15 ++++++---- 4 files changed, 32 insertions(+), 37 deletions(-) diff --git a/sonar/permissions/permissions.py b/sonar/permissions/permissions.py index 142bcbbe9..666a278d0 100644 --- a/sonar/permissions/permissions.py +++ b/sonar/permissions/permissions.py @@ -81,16 +81,15 @@ def __init__(self, concerned_object: object) -> None: def __str__(self) -> str: return f"permissions of {str(self.concerned_object)}" - def to_json(self, perm_type: str = None, csv: bool = False) -> types.JsonPermissions: + def to_json(self, perm_type: str | None = None, csv: bool = False) -> types.JsonPermissions: """Converts a permission object to JSON""" if not csv: - return self.permissions[perm_type] if is_valid(perm_type) else self.permissions + return self.permissions.get(perm_type, {}) if is_valid(perm_type) else self.permissions perms = {} - perm_types = normalize(perm_type) - for p in perm_types: - dperms = self.permissions.get(p, None) - if dperms is not None and len(dperms) > 0: - perms[p] = simplify(dperms) + for p in normalize(perm_type): + if p not in self.permissions or len(self.permissions[p]) == 0: + continue + perms[p] = {k: encode(v) for k, v in self.permissions.get(p, {}).items()} return perms if len(perms) > 0 else None def export(self, export_settings: types.ConfigSettings) -> types.ObjectJsonRepr: @@ -302,7 +301,7 @@ def is_valid(perm_type: str) -> bool: return perm_type and perm_type in PERMISSION_TYPES -def normalize(perm_type: str) -> tuple[str]: +def normalize(perm_type: str | None) -> tuple[str]: """ :meta private: """ @@ -334,29 +333,21 @@ def diff_full(perms_1: types.JsonPermissions, perms_2: types.JsonPermissions) -> def diff(perms_1: types.JsonPermissions, perms_2: types.JsonPermissions) -> types.JsonPermissions: - """ + """Performs the difference between two permissions dictionaries :meta private: """ - diff_perms = perms_1.copy() - for elem, perms in perms_2.items(): - if elem not in perms_1: - continue - for p in perms: - if p not in diff_perms[elem]: - continue - diff_perms[elem].remove(p) - return diff_perms + if not perms_1: + return {} + if not perms_2: + return perms_1 + return {p: diffarray(perms_1[p], perms_2.get(p, [])) for p in perms_1} def diffarray(perms_1: list[str], perms_2: list[str]) -> list[str]: """ :meta private: """ - diff_perms = perms_1.copy() - for elem in perms_2: - if elem in diff_perms: - diff_perms.remove(elem) - return diff_perms + return list(set(perms_1) - set(perms_2)) def white_list(perms: types.JsonPermissions, allowed_perms: list[str]) -> types.JsonPermissions: diff --git a/sonar/permissions/project_permissions.py b/sonar/permissions/project_permissions.py index 82fd68d24..797d2c111 100644 --- a/sonar/permissions/project_permissions.py +++ b/sonar/permissions/project_permissions.py @@ -61,6 +61,7 @@ def read(self) -> ProjectPermissions: # Hack: SonarQube returns application/portfoliocreator even for objects that don't have this permission # so these perms needs to be removed manually self.white_list(tuple(PROJECT_PERMISSIONS.keys())) + self.permissions = {p: k for p, k in self.permissions.items() if k and len(k) > 0} return self def _set_perms( @@ -70,11 +71,9 @@ def _set_perms( if self.permissions is None: self.read() for p in permissions.PERMISSION_TYPES: - if new_perms is None or p not in new_perms: - continue - to_remove = diff_func(self.permissions[p], new_perms[p]) + to_remove = diff_func(self.permissions.get(p, {}), new_perms.get(p, {})) self._post_api(apis["remove"][p], field[p], to_remove, **kwargs) - to_add = diff_func(new_perms[p], self.permissions[p]) + to_add = diff_func(new_perms.get(p, {}), self.permissions.get(p, {})) self._post_api(apis["add"][p], field[p], to_add, **kwargs) return self.read() diff --git a/test/unit/test_apps.py b/test/unit/test_apps.py index d04534a79..e110446ba 100644 --- a/test/unit/test_apps.py +++ b/test/unit/test_apps.py @@ -267,8 +267,8 @@ def test_app_branches(get_test_app: Generator[applications.Application]) -> None app = get_test_app definition = { "branches": { - "Other Branch": {"projects": {"TESTSYNC": "some-branch", "demo:jcl": "main", "training:security": "main"}}, - "BRANCH foo": {"projects": {"TESTSYNC": "some-branch", "demo:jcl": "main", "training:security": "main"}, "isMain": True}, + "Other Branch": {"projects": {"TESTSYNC": "some-branch", "demo:jcl": "main", "demo:java-security": "main"}}, + "BRANCH foo": {"projects": {"TESTSYNC": "some-branch", "demo:jcl": "main", "demo:java-security": "main"}, "isMain": True}, } } app.update(definition) @@ -277,9 +277,9 @@ def test_app_branches(get_test_app: Generator[applications.Application]) -> None assert app.main_branch().name == "BRANCH foo" definition = { "branches": { - "MiBranch": {"projects": {"TESTSYNC": "main", "demo:jcl": "main", "training:security": "main"}}, - "Master": {"projects": {"TESTSYNC": "some-branch", "demo:jcl": "main", "training:security": "main"}}, - "Main Branch": {"projects": {"TESTSYNC": "some-branch", "demo:jcl": "main", "training:security": "main"}, "isMain": True}, + "MiBranch": {"projects": {"TESTSYNC": "main", "demo:jcl": "main", "demo:java-security": "main"}}, + "Master": {"projects": {"TESTSYNC": "some-branch", "demo:jcl": "main", "demo:java-security": "main"}}, + "Main Branch": {"projects": {"TESTSYNC": "some-branch", "demo:jcl": "main", "demo:java-security": "main"}, "isMain": True}, } } app.update(definition) diff --git a/test/unit/test_tasks.py b/test/unit/test_tasks.py index 50387f140..9347502a2 100644 --- a/test/unit/test_tasks.py +++ b/test/unit/test_tasks.py @@ -22,7 +22,7 @@ """ Test of the tasks module and class """ import utilities as tutil -from sonar import tasks +from sonar import tasks, logging def test_task() -> None: @@ -64,6 +64,8 @@ def test_no_scanner_context() -> None: """test_no_scanner_context""" tutil.start_logging() task = tasks.search_last(component_key="project1", endpoint=tutil.SQ, type="REPORT") + if not task: + return if tutil.SQ.version() >= (10, 0, 0): assert task.scanner_context() is None settings = {} @@ -78,11 +80,14 @@ def test_search_all_task() -> None: def test_suspicious_patterns() -> None: """test_suspicious_patterns""" pats = "\\*\\*/[^\/]+/\\*\\*, \\*\\*/\\*[\.\w]*, \\*\\*/\\*, \\*\\*/\\*\\.(java|jav|cs|csx|py|php|js|ts|sql|html|css|cpp|c|h|hpp)\\*?" - assert tasks._get_suspicious_exclusions(pats) == [s.strip() for s in pats.split(",")] - assert tasks._get_suspicious_exclusions(None) == [s.strip() for s in pats.split(",")] + s_pats = tasks._get_suspicious_exclusions(pats) + l_pats = [s.strip() for s in pats.split(",")] + logging.debug(f"{s_pats} == {l_pats}") + assert set(tasks._get_suspicious_exclusions(pats)) == set([s.strip() for s in pats.split(",")]) + assert set(tasks._get_suspicious_exclusions(None)) == set([s.strip() for s in pats.split(",")]) pats = "\\*\\*/(__pycache__|libs|lib|vendor|node_modules)/\\*\\*" - assert tasks._get_suspicious_exceptions(pats) == [s.strip() for s in pats.split(",")] - assert tasks._get_suspicious_exceptions(None) == [s.strip() for s in pats.split(",")] + assert set(tasks._get_suspicious_exceptions(pats)) == set([s.strip() for s in pats.split(",")]) + assert set(tasks._get_suspicious_exceptions(None)) == set([s.strip() for s in pats.split(",")]) # Test does not work - You can't request branch master when scan happened without the branch spec From f3d87d68c0be8f489a310aa523d618370db15f5c Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Fri, 28 Mar 2025 11:15:59 +0100 Subject: [PATCH 24/24] Split-test-99-lts-latest (#1602) * Update sonar-env with new variables * Update specific env test generation * Update env specific environments credentials * Update docker tags for each env * Ignore new unit test temp dirs * Change list of credentials * Change ports * Add logs * Redact tokens in test logs * Adapt test search by large for versions < 10.4 * Redact target tokens too * Fix test for 9.9 --- .gitignore | 10 ++- conf/prep_tests.sh | 3 +- test/integration/it.sh | 31 ++++---- test/sonar-env.sh | 75 +++++++++---------- test/unit/credentials-9-ce.py | 25 +++++++ test/unit/credentials-9.py | 25 +++++++ ...entials-latest-ce.py => credentials-cb.py} | 2 +- ...ntials-lts-ce.py => credentials-lts-de.py} | 2 +- test/unit/credentials-lts.py | 2 +- test/unit/test_issues.py | 7 +- test/unit/test_qg.py | 14 ++-- test/unit/utilities.py | 26 +++++-- 12 files changed, 148 insertions(+), 74 deletions(-) create mode 100644 test/unit/credentials-9-ce.py create mode 100644 test/unit/credentials-9.py rename test/unit/{credentials-latest-ce.py => credentials-cb.py} (95%) rename test/unit/{credentials-lts-ce.py => credentials-lts-de.py} (95%) diff --git a/.gitignore b/.gitignore index 0434eb4e0..1ecfe366a 100644 --- a/.gitignore +++ b/.gitignore @@ -24,9 +24,11 @@ tmp/ !test/files/* .sonar/ test/lts/ +test/lts-de/ test/latest/ -test/cloud/ -test/latest-ce/ -test/lts-ce/ test/latest-de/ -test/lts-de/ +test/cb/ +test/9-ce/ +test/9-de/ +test/9/ +test/cloud/ diff --git a/conf/prep_tests.sh b/conf/prep_tests.sh index 699987ee6..3d212c0b4 100755 --- a/conf/prep_tests.sh +++ b/conf/prep_tests.sh @@ -26,8 +26,9 @@ cd "$ROOTDIR/test/unit" || exit 1 echo "" echo "Generating edition / version specific tests" -for target in lts latest latest-ce lts-ce +for target in lts latest cb 9 9-ce do + echo "Generating tests for $target" rm -rf "$ROOTDIR/test/$target" mkdir -p "$ROOTDIR/test/$target" 2>/dev/null for f in *.py diff --git a/test/integration/it.sh b/test/integration/it.sh index 84178defa..76c9e6eb2 100755 --- a/test/integration/it.sh +++ b/test/integration/it.sh @@ -28,17 +28,23 @@ IT_TEST_PORT=9888 function backup_for { case $1 in - lts|lta|lts-ce|lta-ce|lts-de|lta-de) + lts|lta|lta-ce|lts-de|lta-de) db="$DB_BACKUPS_DIR/db.lts.backup" ;; - lts-audit|lta-audit|lts-audit-ce|lta-audit-ce|lts-audit-de|lta-audit-de) - db="$DB_BACKUPS_DIR/db.lts-audit.backup" + 9|9-de|9-ce) + db="$DB_BACKUPS_DIR/db.9.backup" ;; - latest|latest-ce|latest-de|latest-audit|latest-audit-ce|latest-audit-de) + 9-audit|9-audit-de|9-audit-ce) + db="$DB_BACKUPS_DIR/db.9-audit.backup" + ;; + cb|cb-audit) + db="$DB_BACKUPS_DIR/db.cb.backup" + ;; + latest|latest-de|latest-audit|latest-audit-de) db="$DB_BACKUPS_DIR/db.latest.backup" ;; *) - logmsg "ERROR: Instance $1 has no correspon ding DB backup" + logmsg "ERROR: Instance $1 has no corresponding DB backup" db="NO_DB_BACKUP" esac echo $db @@ -47,22 +53,19 @@ function backup_for { function tag_for { case $1 in lts|lta|lts-audit|lta-audit) - tag="lts-enterprise" - ;; - lts-ce|lta-ce|lts-audit-ce|lta-audit-ce) - tag="lts-community" - ;; - lts-de|lta-de|lts-de-audit|lta-de-audit) - tag="lts-developer" + tag="enterprise" ;; latest|latest-audit) tag="enterprise" ;; + lts-de|lta-de|lts-de-audit|lta-de-audit) + tag="developer" + ;; latest-de|latest-audit-de) tag="developer" ;; - latest-ce|latest-audit-ce) - tag="latest" + cb|cb-audit) + tag="community" ;; *) logmsg "ERROR: Instance $1 has no corresponding tag" diff --git a/test/sonar-env.sh b/test/sonar-env.sh index 0f2c43fc8..aa83bb0ec 100755 --- a/test/sonar-env.sh +++ b/test/sonar-env.sh @@ -1,44 +1,41 @@ -#!/usr/bin/bash +#!/bin/bash source ~/.sonar_rc case $1 in - lts) - export SONAR_TOKEN=$SONAR_TOKEN_LTS_ADMIN_USER - export SONAR_TOKEN_ADMIN=$SONAR_TOKEN_LTS_ADMIN_USER - export SONAR_HOST_URL=$SONAR_HOST_URL_LTS - # $SONAR_HOST_URL_LTS - ;; - lts-user) - export SONAR_TOKEN=$SONAR_TOKEN_LTS_ADMIN_USER - export SONAR_TOKEN_ADMIN=$SONAR_TOKEN_LTS_ADMIN_USER - export SONAR_HOST_URL=$SONAR_HOST_URL_LTS - # $SONAR_HOST_URL_LTS - ;; - lts-audit) - export SONAR_TOKEN=$SONAR_TOKEN_LTS_ADMIN_ANALYSIS - export SONAR_TOKEN_ADMIN=$SONAR_TOKEN_LTS_ADMIN_USER - export SONAR_HOST_URL=http://localhost:9800 - ;; - latest) - export SONAR_TOKEN=$SONAR_TOKEN_LATEST_ADMIN_USER - export SONAR_TOKEN_ADMIN=$SONAR_TOKEN_LATEST_ADMIN_USER - export SONAR_HOST_URL=$SONAR_HOST_URL_LATEST - ;; - latest-user) - export SONAR_TOKEN=$SONAR_TOKEN_LATEST_ADMIN_USER - export SONAR_TOKEN_ADMIN=$SONAR_TOKEN_LATEST_ADMIN_USER - export SONAR_HOST_URL=$SONAR_HOST_URL_LATEST - ;; - nautilus) - export SONAR_TOKEN=$SONAR_TOKEN_NAUTILUS - export SONAR_HOST_URL=$SONAR_HOST_URL_NAUTILUS - ;; - sonarcloud) - export SONAR_TOKEN=$SONAR_TOKEN_SONARCLOUD - export SONAR_HOST_URL=$SONAR_HOST_URL_SONARCLOUD - ;; - *) - echo -u2 "Unsupported sonar environment $1 - Chose between latest lts or nautilus" - ;; + 9) + url=$SONAR_HOST_URL_9 + ;; + 9-ce) + url=$SONAR_HOST_URL_9_CE + ;; + 9-de) + url=$SONAR_HOST_URL_9_DE + ;; + lts) + url=$SONAR_HOST_URL_LTS + ;; + lts-de) + url=$SONAR_HOST_URL_LTS_DE + ;; + latest) + url=$SONAR_HOST_URL_LATEST + ;; + latest-de) + url=$SONAR_HOST_URL_LATEST_DE + ;; + cb) + url=$SONAR_HOST_URL_CB + ;; + cloud) + url="https://sonarcloud.io" + ;; + *) + echo "Error: Usage: $0 [lts|latest|lts-de|latest-de|cb|9|9-de|9-ce|cloud]" + return + ;; esac + +export SONAR_HOST_URL=$url + +echo "SONAR_HOST_URL=$url" diff --git a/test/unit/credentials-9-ce.py b/test/unit/credentials-9-ce.py new file mode 100644 index 000000000..f12ea8112 --- /dev/null +++ b/test/unit/credentials-9-ce.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +# +# sonar-tools tests +# Copyright (C) 2025 Olivier Korach +# mailto:olivier.korach AT gmail DOT com +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3 of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# + +from os import getenv + +TARGET_PLATFORM = "http://localhost:9002" +TARGET_TOKEN = getenv("SONAR_TOKEN_9_ADMIN_USER") diff --git a/test/unit/credentials-9.py b/test/unit/credentials-9.py new file mode 100644 index 000000000..11f0f4ae9 --- /dev/null +++ b/test/unit/credentials-9.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +# +# sonar-tools tests +# Copyright (C) 2025 Olivier Korach +# mailto:olivier.korach AT gmail DOT com +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3 of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# + +from os import getenv + +TARGET_PLATFORM = "http://localhost:9000" +TARGET_TOKEN = getenv("SONAR_TOKEN_9_ADMIN_USER") diff --git a/test/unit/credentials-latest-ce.py b/test/unit/credentials-cb.py similarity index 95% rename from test/unit/credentials-latest-ce.py rename to test/unit/credentials-cb.py index 96b6cfd18..74abc2d07 100644 --- a/test/unit/credentials-latest-ce.py +++ b/test/unit/credentials-cb.py @@ -21,5 +21,5 @@ from os import getenv -TARGET_PLATFORM = "http://localhost:10001" +TARGET_PLATFORM = "http://localhost:7000" TARGET_TOKEN = getenv("SONAR_TOKEN_LATEST_ADMIN_USER") diff --git a/test/unit/credentials-lts-ce.py b/test/unit/credentials-lts-de.py similarity index 95% rename from test/unit/credentials-lts-ce.py rename to test/unit/credentials-lts-de.py index e00cca5c3..e1d0d18b2 100644 --- a/test/unit/credentials-lts-ce.py +++ b/test/unit/credentials-lts-de.py @@ -21,5 +21,5 @@ from os import getenv -TARGET_PLATFORM = "http://localhost:9001" +TARGET_PLATFORM = "http://localhost:8001" TARGET_TOKEN = getenv("SONAR_TOKEN_LTS_ADMIN_USER") diff --git a/test/unit/credentials-lts.py b/test/unit/credentials-lts.py index 4407fe781..a0ac17eac 100644 --- a/test/unit/credentials-lts.py +++ b/test/unit/credentials-lts.py @@ -21,5 +21,5 @@ from os import getenv -TARGET_PLATFORM = "http://localhost:9000" +TARGET_PLATFORM = "http://localhost:8000" TARGET_TOKEN = getenv("SONAR_TOKEN_LTS_ADMIN_USER") diff --git a/test/unit/test_issues.py b/test/unit/test_issues.py index 3746dfc4b..eae7cb7f0 100644 --- a/test/unit/test_issues.py +++ b/test/unit/test_issues.py @@ -232,5 +232,8 @@ def test_search_by_large() -> None: assert len(issues.search_by_project(tutil.SQ, "pytorch")) > 10000 params = {"components": "pytorch", "project": "pytorch"} - with pytest.raises(issues.TooManyIssuesError): - issues.search_by_severity(tutil.SQ, params) + + # Versions below 10.4 did not have enough python rules to break the 10K limit on the pytorch project + if tutil.SQ.version() >= (10, 4, 0): + with pytest.raises(issues.TooManyIssuesError): + issues.search_by_severity(tutil.SQ, params) diff --git a/test/unit/test_qg.py b/test/unit/test_qg.py index d38fe61dc..fdf23b92e 100644 --- a/test/unit/test_qg.py +++ b/test/unit/test_qg.py @@ -34,10 +34,12 @@ def test_get_object(get_loaded_qg: Generator[qualitygates.QualityGate]) -> None: qg = get_loaded_qg assert qg.name == util.TEMP_KEY assert str(qg) == f"quality gate '{util.TEMP_KEY}'" - assert qg.url() == f"{util.SQ.url}/quality_gates/show/{util.TEMP_KEY}" + if util.SQ.version() < (10, 0, 0): + assert qg.url() == f"{util.SQ.url}/quality_gates/show/{qg.key}" + else: + assert qg.url() == f"{util.SQ.url}/quality_gates/show/{util.TEMP_KEY}" qg2 = qualitygates.QualityGate.get_object(endpoint=util.SQ, name=util.TEMP_KEY) assert qg.projects() == {} - assert qg.projects() == {} assert qg2 is qg @@ -143,12 +145,12 @@ def test_audit(get_empty_qg: Generator[qualitygates.QualityGate]) -> None: "new_coverage <= 80", "new_duplicated_lines_density >= 3", "new_security_hotspots_reviewed <= 100", - "new_software_quality_blocker_issues >= 0", - "new_software_quality_high_issues >= 0", - "new_software_quality_medium_issues >= 0", + "new_technical_debt >= 1000", + "comment_lines_density <= 10", + "coverage <= 20", ] qg.set_conditions(conds) - assert len(qg.audit({"audit.qualitygates.maxConditions": 5})) == 2 + assert len(qg.audit({"audit.qualitygates.maxConditions": 5})) >= 2 def test_count(): diff --git a/test/unit/utilities.py b/test/unit/utilities.py index 964a30b64..08cd11018 100644 --- a/test/unit/utilities.py +++ b/test/unit/utilities.py @@ -32,6 +32,7 @@ import credentials as creds from sonar import errcodes, logging +from sonar import utilities as util from sonar import platform import cli.options as opt @@ -40,9 +41,12 @@ FILES_ROOT = "test/files/" LATEST = "http://localhost:10000" -LATEST_TEST = "http://localhost:10020" -LTA = "http://localhost:9000" -LATEST_CE = "http://localhost:8000" +LTA = "http://localhost:8000" +LTS = LTA + +LATEST_TEST = "http://localhost:10010" + +CB = "http://localhost:7000" TARGET_PLATFORM = LATEST @@ -71,7 +75,7 @@ TEST_OPTS = [f"-{opt.URL_SHORT}", LATEST_TEST, f"-{opt.TOKEN_SHORT}", os.getenv("SONAR_TOKEN_ADMIN_USER")] SQS_TEST_OPTS = " ".join(TEST_OPTS) -CE_OPTS = [f"-{opt.URL_SHORT}", LATEST_CE, f"-{opt.TOKEN_SHORT}", os.getenv("SONAR_TOKEN_ADMIN_USER")] +CE_OPTS = [f"-{opt.URL_SHORT}", CB, f"-{opt.TOKEN_SHORT}", os.getenv("SONAR_TOKEN_ADMIN_USER")] SC_OPTS = f'--{opt.URL} https://sonarcloud.io --{opt.TOKEN} {os.getenv("SONAR_TOKEN_SONARCLOUD")} --{opt.ORG} okorach' @@ -147,10 +151,22 @@ def __get_args_and_file(string_arguments: str) -> tuple[Optional[str], list[str] file = None return file, args +def __get_redacted_cmd(string_arguments: str) -> str: + """Gets a cmd line and redacts the token""" + args = string_arguments.split(" ") + + for option in (f"-{opt.TOKEN_SHORT}", f"--{opt.TOKEN}", f"-T", f"--tokenTarget"): + try: + ndx = args.index(f"{option}") + 1 + args[ndx] = util.redacted_token(args[ndx]) + except ValueError: + pass + return " ".join(args) + def run_cmd(func: callable, arguments: str, expected_code: int) -> Optional[str]: """Runs a sonar-tools command, verifies it raises the right exception, and returns the expected code""" - logging.info("RUNNING: %s", arguments) + logging.info("RUNNING: %s", __get_redacted_cmd(arguments)) file, args = __get_args_and_file(arguments) with pytest.raises(SystemExit) as e: with patch.object(sys, "argv", args):