Python package license detection

lib/cyclonedx/report_python_modules_cyclonedx.rb

@@ -7,6 +7,18 @@ def initialize(scan_report)
       super(scan_report)
     end
 
+    def build_component(dependency)
+      super.merge({
+                    "licenses": licenses_for(dependency)
+                  })
+    end
+
+    def licenses_for(dependency)
+      return [] if dependency[:licenses].nil?
+
+      dependency[:licenses].map { |license| { "license": { "id" => license } } }
+    end
+
     # Return version string to be used in purl or component
     def version_string(dependency, is_purl_version = false)
       # Check if dependency is pinned

lib/salus/scanners/report_python_modules.rb

@@ -1,26 +1,33 @@
+require 'uri'
+require 'net/http'
 require 'salus/scanners/report_base'
 
 # Report python library usage
 
 module Salus::Scanners
   class ReportPythonModules < ReportBase
+    class PyPiApiError < StandardError; end
+    class ApiTooManyRequestsError < StandardError; end
+
+    MAX_RETRIES_FOR_API = 2
+
+    def self.scanner_type
+      Salus::ScannerTypes::SBOM_REPORT
+    end
+
     def run
       shell_return = run_shell(['bin/report_python_modules',
                                 @repository.path_to_repo], chdir: nil)
 
-      if !shell_return.success?
-        report_error(shell_return.stderr)
-        return
-      end
-
       dependencies = JSON.parse(shell_return.stdout)
 
       dependencies.each do |name, version|
         report_dependency(
           'requirements.txt',
           type: 'pypi',
           name: name,
-          version: version
+          version: version,
+          licenses: find_licenses_for(name)
         )
       end
     end
@@ -32,5 +39,56 @@ def should_run?
     def self.supported_languages
       ['python']
     end
+
+    private
+
+    def find_licenses_for(name)
+      res = send_request_to(name)
+
+      return [] if res.nil?
+      return [] if res.is_a?(Net::HTTPNotFound)
+
+      raise PyPiApiError, res.body unless res.is_a?(Net::HTTPSuccess)
+
+      licenses = JSON.parse(res.body).dig("info", "license")
+      [licenses]
+    rescue PyPiApiError, StandardError => e
+      msg = "Unable to gather license information " \
+        "using pypi api " \
+        "with error message #{e.class}: #{e.message}"
+      bugsnag_notify(msg)
+
+      []
+    end
+
+    def send_request_to(name)
+      retries = 0
+
+      begin
+        uri = pypi_uri_for(name)
+        res = Net::HTTP.get_response(uri)
+
+        raise ApiTooManyRequestsError if res.is_a?(Net::HTTPTooManyRequests)
+
+        res
+      rescue ApiTooManyRequestsError
+        if retries < MAX_RETRIES_FOR_API
+          retries += 1
+          max_sleep_seconds = Float(2**retries)
+          sleep rand(0..max_sleep_seconds)
+          retry
+        else
+          msg = "Too many requests for pypi api after " \
+            "#{retries} retries"
+          bugsnag_notify(msg)
+
+          nil
+        end
+      end
+    end
+
+    def pypi_uri_for(name)
+      URI("https://pypi.org/pypi/#{name}/json")
+    end
   end
 end

spec/lib/cyclonedx/report_python_modules_cyclonedx_spec.rb

@@ -2,6 +2,12 @@
 require 'json'
 
 describe Cyclonedx::ReportPythonModules do
+  before do
+    allow_any_instance_of(Salus::Scanners::ReportPythonModules)
+      .to receive(:find_licenses_for)
+      .and_return(['MIT'])
+  end
+
   describe "#run" do
     it 'should report all the deps in the unpinned requirements.txt' do
       repo = Salus::Repo.new('spec/fixtures/python/requirements_unpinned')
@@ -13,20 +19,23 @@
         {
           "type": "library",
           "group": "",
+          "licenses": [{ "license": { "id" => "MIT" } }],
           "name": "requests",
           "version": ">=2.5",
           "purl": "pkg:pypi/requests"
         },
         {
           "type": "library",
           "group": "",
+            "licenses": [{ "license": { "id" => "MIT" } }],
             "name": "six",
             "version": ">=1.9",
             "purl": "pkg:pypi/six"
         },
         {
           "type": "library",
           "group": "",
+            "licenses": [{ "license": { "id" => "MIT" } }],
             "name": "pycryptodome",
             "version": ">=3.4.11",
             "purl": "pkg:pypi/pycryptodome"
@@ -45,20 +54,23 @@
         {
           "type": "library",
           "group": "",
+          "licenses": [{ "license": { "id" => "MIT" } }],
           "name": "amqp",
           "version": "2.2.2",
           "purl": "pkg:pypi/amqp@2.2.2"
         },
         {
           "type": "library",
           "group": "",
+          "licenses": [{ "license": { "id" => "MIT" } }],
           "name": "billiard",
           "version": "3.5.0.3",
           "purl": "pkg:pypi/billiard@3.5.0.3"
         },
         {
           "type": "library",
           "group": "",
+          "licenses": [{ "license": { "id" => "MIT" } }],
           "name": "celery",
           "version": "4.1.0",
           "purl": "pkg:pypi/celery@4.1.0"
@@ -77,41 +89,47 @@
         {
           "type": "library",
           "group": "",
+          "licenses": [{ "license": { "id" => "MIT" } }],
           "name": "six",
           "version": ">=1.9",
           "purl": "pkg:pypi/six"
         },
         {
           "type": "library",
           "group": "",
+          "licenses": [{ "license": { "id" => "MIT" } }],
           "name": "pycryptodome",
           "version": ">=3.4.11",
           "purl": "pkg:pypi/pycryptodome"
         },
         {
           "type": "library",
           "group": "",
+          "licenses": [{ "license": { "id" => "MIT" } }],
           "name": "celery",
           "version": "4.0.0",
           "purl": "pkg:pypi/celery@4.0.0"
         },
         {
           "type": "library",
           "group": "",
+          "licenses": [{ "license": { "id" => "MIT" } }],
           "name": "Jinja2",
           "version": "2.10",
           "purl": "pkg:pypi/Jinja2@2.10"
         },
         {
           "type": "library",
           "group": "",
+          "licenses": [{ "license": { "id" => "MIT" } }],
           "name": "itsdangerous",
           "version": "0.24",
           "purl": "pkg:pypi/itsdangerous@0.24"
         },
         {
           "type": "library",
           "group": "",
+          "licenses": [{ "license": { "id" => "MIT" } }],
           "name": "idna",
           "version": "2.6",
           "purl": "pkg:pypi/idna@2.6"

spec/lib/salus/scanners/report_python_modules_spec.rb

@@ -1,6 +1,10 @@
 require_relative '../../../spec_helper.rb'
 
 describe Salus::Scanners::ReportPythonModules do
+  before do
+    allow_any_instance_of(described_class).to receive(:find_licenses_for).and_return(['MIT'])
+  end
+
   describe '#should_run?' do
     it 'should return false in the absence of requirements.txt' do
       repo = Salus::Repo.new('spec/fixtures/blank_repository')
@@ -29,19 +33,22 @@
           dependency_file: 'requirements.txt',
           name: 'requests',
           version: '>=2.5',
-          type: 'pypi'
+          type: 'pypi',
+          licenses: ['MIT']
         },
         {
           dependency_file: 'requirements.txt',
           name: 'six',
           version: '>=1.9',
-          type: 'pypi'
+          type: 'pypi',
+          licenses: ['MIT']
         },
         {
           dependency_file: 'requirements.txt',
           name: 'pycryptodome',
           version: '>=3.4.11',
-          type: 'pypi'
+          type: 'pypi',
+          licenses: ['MIT']
         }
       ]
     )