diff --git a/lib/cyclonedx/report_python_modules_cyclonedx.rb b/lib/cyclonedx/report_python_modules_cyclonedx.rb index 37341d7a..105accad 100644 --- a/lib/cyclonedx/report_python_modules_cyclonedx.rb +++ b/lib/cyclonedx/report_python_modules_cyclonedx.rb @@ -7,6 +7,18 @@ def initialize(scan_report) super(scan_report) end + def build_component(dependency) + super.merge({ + "licenses": licenses_for(dependency) + }) + end + + def licenses_for(dependency) + return [] if dependency[:licenses].nil? + + dependency[:licenses].map { |license| { "license": { "id" => license } } } + end + # Return version string to be used in purl or component def version_string(dependency, is_purl_version = false) # Check if dependency is pinned diff --git a/lib/salus/scanners/report_python_modules.rb b/lib/salus/scanners/report_python_modules.rb index f6e1f2f8..6b08b600 100644 --- a/lib/salus/scanners/report_python_modules.rb +++ b/lib/salus/scanners/report_python_modules.rb @@ -1,18 +1,24 @@ +require 'uri' +require 'net/http' require 'salus/scanners/report_base' # Report python library usage module Salus::Scanners class ReportPythonModules < ReportBase + class PyPiApiError < StandardError; end + class ApiTooManyRequestsError < StandardError; end + + MAX_RETRIES_FOR_API = 2 + + def self.scanner_type + Salus::ScannerTypes::SBOM_REPORT + end + def run shell_return = run_shell(['bin/report_python_modules', @repository.path_to_repo], chdir: nil) - if !shell_return.success? - report_error(shell_return.stderr) - return - end - dependencies = JSON.parse(shell_return.stdout) dependencies.each do |name, version| @@ -20,7 +26,8 @@ def run 'requirements.txt', type: 'pypi', name: name, - version: version + version: version, + licenses: find_licenses_for(name) ) end end @@ -32,5 +39,56 @@ def should_run? def self.supported_languages ['python'] end + + private + + def find_licenses_for(name) + res = send_request_to(name) + + return [] if res.nil? + return [] if res.is_a?(Net::HTTPNotFound) + + raise PyPiApiError, res.body unless res.is_a?(Net::HTTPSuccess) + + licenses = JSON.parse(res.body).dig("info", "license") + [licenses] + rescue PyPiApiError, StandardError => e + msg = "Unable to gather license information " \ + "using pypi api " \ + "with error message #{e.class}: #{e.message}" + bugsnag_notify(msg) + + [] + end + + def send_request_to(name) + retries = 0 + + begin + uri = pypi_uri_for(name) + res = Net::HTTP.get_response(uri) + + raise ApiTooManyRequestsError if res.is_a?(Net::HTTPTooManyRequests) + + res + rescue ApiTooManyRequestsError + if retries < MAX_RETRIES_FOR_API + retries += 1 + max_sleep_seconds = Float(2**retries) + sleep rand(0..max_sleep_seconds) + retry + else + msg = "Too many requests for pypi api after " \ + "#{retries} retries" + bugsnag_notify(msg) + + nil + end + end + end + + def pypi_uri_for(name) + URI("https://pypi.org/pypi/#{name}/json") + end end end diff --git a/spec/lib/cyclonedx/report_python_modules_cyclonedx_spec.rb b/spec/lib/cyclonedx/report_python_modules_cyclonedx_spec.rb index 5cbaa955..11b3c6a6 100644 --- a/spec/lib/cyclonedx/report_python_modules_cyclonedx_spec.rb +++ b/spec/lib/cyclonedx/report_python_modules_cyclonedx_spec.rb @@ -2,6 +2,12 @@ require 'json' describe Cyclonedx::ReportPythonModules do + before do + allow_any_instance_of(Salus::Scanners::ReportPythonModules) + .to receive(:find_licenses_for) + .and_return(['MIT']) + end + describe "#run" do it 'should report all the deps in the unpinned requirements.txt' do repo = Salus::Repo.new('spec/fixtures/python/requirements_unpinned') @@ -13,6 +19,7 @@ { "type": "library", "group": "", + "licenses": [{ "license": { "id" => "MIT" } }], "name": "requests", "version": ">=2.5", "purl": "pkg:pypi/requests" @@ -20,6 +27,7 @@ { "type": "library", "group": "", + "licenses": [{ "license": { "id" => "MIT" } }], "name": "six", "version": ">=1.9", "purl": "pkg:pypi/six" @@ -27,6 +35,7 @@ { "type": "library", "group": "", + "licenses": [{ "license": { "id" => "MIT" } }], "name": "pycryptodome", "version": ">=3.4.11", "purl": "pkg:pypi/pycryptodome" @@ -45,6 +54,7 @@ { "type": "library", "group": "", + "licenses": [{ "license": { "id" => "MIT" } }], "name": "amqp", "version": "2.2.2", "purl": "pkg:pypi/amqp@2.2.2" @@ -52,6 +62,7 @@ { "type": "library", "group": "", + "licenses": [{ "license": { "id" => "MIT" } }], "name": "billiard", "version": "3.5.0.3", "purl": "pkg:pypi/billiard@3.5.0.3" @@ -59,6 +70,7 @@ { "type": "library", "group": "", + "licenses": [{ "license": { "id" => "MIT" } }], "name": "celery", "version": "4.1.0", "purl": "pkg:pypi/celery@4.1.0" @@ -77,6 +89,7 @@ { "type": "library", "group": "", + "licenses": [{ "license": { "id" => "MIT" } }], "name": "six", "version": ">=1.9", "purl": "pkg:pypi/six" @@ -84,6 +97,7 @@ { "type": "library", "group": "", + "licenses": [{ "license": { "id" => "MIT" } }], "name": "pycryptodome", "version": ">=3.4.11", "purl": "pkg:pypi/pycryptodome" @@ -91,6 +105,7 @@ { "type": "library", "group": "", + "licenses": [{ "license": { "id" => "MIT" } }], "name": "celery", "version": "4.0.0", "purl": "pkg:pypi/celery@4.0.0" @@ -98,6 +113,7 @@ { "type": "library", "group": "", + "licenses": [{ "license": { "id" => "MIT" } }], "name": "Jinja2", "version": "2.10", "purl": "pkg:pypi/Jinja2@2.10" @@ -105,6 +121,7 @@ { "type": "library", "group": "", + "licenses": [{ "license": { "id" => "MIT" } }], "name": "itsdangerous", "version": "0.24", "purl": "pkg:pypi/itsdangerous@0.24" @@ -112,6 +129,7 @@ { "type": "library", "group": "", + "licenses": [{ "license": { "id" => "MIT" } }], "name": "idna", "version": "2.6", "purl": "pkg:pypi/idna@2.6" diff --git a/spec/lib/salus/scanners/report_python_modules_spec.rb b/spec/lib/salus/scanners/report_python_modules_spec.rb index 039addc2..4d51b7a5 100644 --- a/spec/lib/salus/scanners/report_python_modules_spec.rb +++ b/spec/lib/salus/scanners/report_python_modules_spec.rb @@ -1,6 +1,10 @@ require_relative '../../../spec_helper.rb' describe Salus::Scanners::ReportPythonModules do + before do + allow_any_instance_of(described_class).to receive(:find_licenses_for).and_return(['MIT']) + end + describe '#should_run?' do it 'should return false in the absence of requirements.txt' do repo = Salus::Repo.new('spec/fixtures/blank_repository') @@ -29,19 +33,22 @@ dependency_file: 'requirements.txt', name: 'requests', version: '>=2.5', - type: 'pypi' + type: 'pypi', + licenses: ['MIT'] }, { dependency_file: 'requirements.txt', name: 'six', version: '>=1.9', - type: 'pypi' + type: 'pypi', + licenses: ['MIT'] }, { dependency_file: 'requirements.txt', name: 'pycryptodome', version: '>=3.4.11', - type: 'pypi' + type: 'pypi', + licenses: ['MIT'] } ] )