From 091c923e659144d380a635b80369fab22d2ba671 Mon Sep 17 00:00:00 2001 From: Maitray Shah Date: Tue, 28 Jun 2022 00:51:24 -0700 Subject: [PATCH 1/5] License detection for Python packages --- Dockerfile | 5 ++ .../report_python_modules_cyclonedx.rb | 12 ++++ lib/salus/scanners/report_python_modules.rb | 65 +++++++++++++++++-- 3 files changed, 76 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index b58abc3c..6ba1ced2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -68,6 +68,11 @@ RUN pip install wheel \ && mv .local/bin/bandit .local/bin/bandit2 \ && pip3 install --user bandit==${BANDIT_VERSION} + +ENV PIPDEPTREE_VERSION 2.2.1 +RUN pip3 install --user pipdeptree==${PIPDEPTREE_VERSION} +RUN pip3 install --user pipgrip + ### Ruby # ruby gems COPY Gemfile Gemfile.lock /home/ diff --git a/lib/cyclonedx/report_python_modules_cyclonedx.rb b/lib/cyclonedx/report_python_modules_cyclonedx.rb index 37341d7a..888065a9 100644 --- a/lib/cyclonedx/report_python_modules_cyclonedx.rb +++ b/lib/cyclonedx/report_python_modules_cyclonedx.rb @@ -7,6 +7,18 @@ def initialize(scan_report) super(scan_report) end + def build_component(dependency) + super.merge({ + "licenses": licenses_for(dependency) + }) + end + + def licenses_for(dependency) + return [] if dependency[:licenses].nil? + + dependency[:licenses].map { |license| { "license" => { "id" => license } } } + end + # Return version string to be used in purl or component def version_string(dependency, is_purl_version = false) # Check if dependency is pinned diff --git a/lib/salus/scanners/report_python_modules.rb b/lib/salus/scanners/report_python_modules.rb index 09581129..155081a6 100644 --- a/lib/salus/scanners/report_python_modules.rb +++ b/lib/salus/scanners/report_python_modules.rb @@ -4,6 +4,11 @@ module Salus::Scanners class ReportPythonModules < Base + class PyPiApiError < StandardError; end + class ApiTooManyRequestsError < StandardError; end + + MAX_RETRIES_FOR_API = 2 + def self.scanner_type Salus::ScannerTypes::SBOM_REPORT end @@ -12,19 +17,16 @@ def run shell_return = run_shell(['bin/report_python_modules', @repository.path_to_repo], chdir: nil) - if !shell_return.success? - report_error(shell_return.stderr) - return - end - dependencies = JSON.parse(shell_return.stdout) dependencies.each do |name, version| + puts find_licenses_for(name) report_dependency( 'requirements.txt', type: 'pypi', name: name, - version: version + version: version, + licenses: find_licenses_for(name) ) end end @@ -36,5 +38,56 @@ def should_run? def self.supported_languages ['python'] end + + private + + def find_licenses_for(name) + res = send_request_to(name) + + return [] if res.nil? + return [] if res.is_a?(Net::HTTPNotFound) + + raise PyPiApiError, res.body unless res.is_a?(Net::HTTPSuccess) + + licenses = JSON.parse(res.body).dig("info", "license") + [licenses] + rescue RubyGemsApiError, StandardError => e + msg = "Unable to gather license information " \ + "using pypi api " \ + "with error message #{e.class}: #{e.message}" + bugsnag_notify(msg) + + [] + end + + def send_request_to(name) + retries = 0 + + begin + uri = pypi_uri_for(name) + res = Net::HTTP.get_response(uri) + + raise ApiTooManyRequestsError if res.is_a?(Net::HTTPTooManyRequests) + + res + rescue ApiTooManyRequestsError + if retries < MAX_RETRIES_FOR_API + retries += 1 + max_sleep_seconds = Float(2**retries) + sleep rand(0..max_sleep_seconds) + retry + else + msg = "Too many requests for pypi api after " \ + "#{retries} retries" + bugsnag_notify(msg) + + nil + end + end + end + + def pypi_uri_for(name) + URI("https://pypi.org/pypi/#{name}/json") + end end end From 67905936163240a0e5ed8bf291293a2b9edf96a0 Mon Sep 17 00:00:00 2001 From: Maitray Shah Date: Tue, 28 Jun 2022 00:52:18 -0700 Subject: [PATCH 2/5] License detection for Python packages --- lib/salus/scanners/report_python_modules.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/salus/scanners/report_python_modules.rb b/lib/salus/scanners/report_python_modules.rb index 155081a6..e63d6666 100644 --- a/lib/salus/scanners/report_python_modules.rb +++ b/lib/salus/scanners/report_python_modules.rb @@ -51,7 +51,7 @@ def find_licenses_for(name) licenses = JSON.parse(res.body).dig("info", "license") [licenses] - rescue RubyGemsApiError, StandardError => e + rescue PyPiApiError, StandardError => e msg = "Unable to gather license information " \ "using pypi api " \ "with error message #{e.class}: #{e.message}" From daec379bc51740e43ea514e29807d7b6fe2c6ff4 Mon Sep 17 00:00:00 2001 From: Maitray Shah Date: Tue, 28 Jun 2022 00:53:55 -0700 Subject: [PATCH 3/5] License detection for Python packages --- lib/salus/scanners/report_python_modules.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/salus/scanners/report_python_modules.rb b/lib/salus/scanners/report_python_modules.rb index e63d6666..05ed573b 100644 --- a/lib/salus/scanners/report_python_modules.rb +++ b/lib/salus/scanners/report_python_modules.rb @@ -1,4 +1,6 @@ require 'salus/scanners/base' +require 'uri' +require 'net/http' # Report python library usage @@ -20,7 +22,6 @@ def run dependencies = JSON.parse(shell_return.stdout) dependencies.each do |name, version| - puts find_licenses_for(name) report_dependency( 'requirements.txt', type: 'pypi', From 01d020b8caaf1c312f85ea695e082a3eec29ec6c Mon Sep 17 00:00:00 2001 From: Maitray Shah Date: Tue, 28 Jun 2022 11:51:59 -0700 Subject: [PATCH 4/5] Remove deps --- Dockerfile | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6ba1ced2..b58abc3c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -68,11 +68,6 @@ RUN pip install wheel \ && mv .local/bin/bandit .local/bin/bandit2 \ && pip3 install --user bandit==${BANDIT_VERSION} - -ENV PIPDEPTREE_VERSION 2.2.1 -RUN pip3 install --user pipdeptree==${PIPDEPTREE_VERSION} -RUN pip3 install --user pipgrip - ### Ruby # ruby gems COPY Gemfile Gemfile.lock /home/ From 980939a29035f31a3e616c73a0e326a0b5ba42b4 Mon Sep 17 00:00:00 2001 From: Maitray Shah Date: Mon, 4 Jul 2022 15:46:12 -0700 Subject: [PATCH 5/5] Test case fix --- .../report_python_modules_cyclonedx.rb | 2 +- .../report_python_modules_cyclonedx_spec.rb | 18 ++++++++++++++++++ .../scanners/report_python_modules_spec.rb | 13 ++++++++++--- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/lib/cyclonedx/report_python_modules_cyclonedx.rb b/lib/cyclonedx/report_python_modules_cyclonedx.rb index 888065a9..105accad 100644 --- a/lib/cyclonedx/report_python_modules_cyclonedx.rb +++ b/lib/cyclonedx/report_python_modules_cyclonedx.rb @@ -16,7 +16,7 @@ def build_component(dependency) def licenses_for(dependency) return [] if dependency[:licenses].nil? - dependency[:licenses].map { |license| { "license" => { "id" => license } } } + dependency[:licenses].map { |license| { "license": { "id" => license } } } end # Return version string to be used in purl or component diff --git a/spec/lib/cyclonedx/report_python_modules_cyclonedx_spec.rb b/spec/lib/cyclonedx/report_python_modules_cyclonedx_spec.rb index 5cbaa955..11b3c6a6 100644 --- a/spec/lib/cyclonedx/report_python_modules_cyclonedx_spec.rb +++ b/spec/lib/cyclonedx/report_python_modules_cyclonedx_spec.rb @@ -2,6 +2,12 @@ require 'json' describe Cyclonedx::ReportPythonModules do + before do + allow_any_instance_of(Salus::Scanners::ReportPythonModules) + .to receive(:find_licenses_for) + .and_return(['MIT']) + end + describe "#run" do it 'should report all the deps in the unpinned requirements.txt' do repo = Salus::Repo.new('spec/fixtures/python/requirements_unpinned') @@ -13,6 +19,7 @@ { "type": "library", "group": "", + "licenses": [{ "license": { "id" => "MIT" } }], "name": "requests", "version": ">=2.5", "purl": "pkg:pypi/requests" @@ -20,6 +27,7 @@ { "type": "library", "group": "", + "licenses": [{ "license": { "id" => "MIT" } }], "name": "six", "version": ">=1.9", "purl": "pkg:pypi/six" @@ -27,6 +35,7 @@ { "type": "library", "group": "", + "licenses": [{ "license": { "id" => "MIT" } }], "name": "pycryptodome", "version": ">=3.4.11", "purl": "pkg:pypi/pycryptodome" @@ -45,6 +54,7 @@ { "type": "library", "group": "", + "licenses": [{ "license": { "id" => "MIT" } }], "name": "amqp", "version": "2.2.2", "purl": "pkg:pypi/amqp@2.2.2" @@ -52,6 +62,7 @@ { "type": "library", "group": "", + "licenses": [{ "license": { "id" => "MIT" } }], "name": "billiard", "version": "3.5.0.3", "purl": "pkg:pypi/billiard@3.5.0.3" @@ -59,6 +70,7 @@ { "type": "library", "group": "", + "licenses": [{ "license": { "id" => "MIT" } }], "name": "celery", "version": "4.1.0", "purl": "pkg:pypi/celery@4.1.0" @@ -77,6 +89,7 @@ { "type": "library", "group": "", + "licenses": [{ "license": { "id" => "MIT" } }], "name": "six", "version": ">=1.9", "purl": "pkg:pypi/six" @@ -84,6 +97,7 @@ { "type": "library", "group": "", + "licenses": [{ "license": { "id" => "MIT" } }], "name": "pycryptodome", "version": ">=3.4.11", "purl": "pkg:pypi/pycryptodome" @@ -91,6 +105,7 @@ { "type": "library", "group": "", + "licenses": [{ "license": { "id" => "MIT" } }], "name": "celery", "version": "4.0.0", "purl": "pkg:pypi/celery@4.0.0" @@ -98,6 +113,7 @@ { "type": "library", "group": "", + "licenses": [{ "license": { "id" => "MIT" } }], "name": "Jinja2", "version": "2.10", "purl": "pkg:pypi/Jinja2@2.10" @@ -105,6 +121,7 @@ { "type": "library", "group": "", + "licenses": [{ "license": { "id" => "MIT" } }], "name": "itsdangerous", "version": "0.24", "purl": "pkg:pypi/itsdangerous@0.24" @@ -112,6 +129,7 @@ { "type": "library", "group": "", + "licenses": [{ "license": { "id" => "MIT" } }], "name": "idna", "version": "2.6", "purl": "pkg:pypi/idna@2.6" diff --git a/spec/lib/salus/scanners/report_python_modules_spec.rb b/spec/lib/salus/scanners/report_python_modules_spec.rb index 039addc2..4d51b7a5 100644 --- a/spec/lib/salus/scanners/report_python_modules_spec.rb +++ b/spec/lib/salus/scanners/report_python_modules_spec.rb @@ -1,6 +1,10 @@ require_relative '../../../spec_helper.rb' describe Salus::Scanners::ReportPythonModules do + before do + allow_any_instance_of(described_class).to receive(:find_licenses_for).and_return(['MIT']) + end + describe '#should_run?' do it 'should return false in the absence of requirements.txt' do repo = Salus::Repo.new('spec/fixtures/blank_repository') @@ -29,19 +33,22 @@ dependency_file: 'requirements.txt', name: 'requests', version: '>=2.5', - type: 'pypi' + type: 'pypi', + licenses: ['MIT'] }, { dependency_file: 'requirements.txt', name: 'six', version: '>=1.9', - type: 'pypi' + type: 'pypi', + licenses: ['MIT'] }, { dependency_file: 'requirements.txt', name: 'pycryptodome', version: '>=3.4.11', - type: 'pypi' + type: 'pypi', + licenses: ['MIT'] } ] )