From cce49e26c1f92c6e18995ef3f1bae09519598a25 Mon Sep 17 00:00:00 2001 From: sachin-sandhu Date: Tue, 10 Dec 2024 10:39:32 -0500 Subject: [PATCH 01/11] adds dotnet with metric collection --- nuget/lib/dependabot/nuget/file_parser.rb | 96 +++++++++++++++++++ nuget/lib/dependabot/nuget/language.rb | 82 ++++++++++++++++ nuget/lib/dependabot/nuget/package_manager.rb | 51 ++++++++++ 3 files changed, 229 insertions(+) create mode 100644 nuget/lib/dependabot/nuget/language.rb create mode 100644 nuget/lib/dependabot/nuget/package_manager.rb diff --git a/nuget/lib/dependabot/nuget/file_parser.rb b/nuget/lib/dependabot/nuget/file_parser.rb index 8022071b37..ffdc37a0da 100644 --- a/nuget/lib/dependabot/nuget/file_parser.rb +++ b/nuget/lib/dependabot/nuget/file_parser.rb @@ -6,6 +6,8 @@ require "dependabot/file_parsers/base" require "dependabot/nuget/native_discovery/native_discovery_json_reader" require "dependabot/nuget/native_helpers" +require "dependabot/nuget/package_manager" +require "dependabot/nuget/language" require "sorbet-runtime" # For details on how dotnet handles version constraints, see: @@ -22,6 +24,18 @@ def parse dependencies end + sig { returns(Ecosystem) } + def ecosystem + @ecosystem ||= T.let( + Ecosystem.new( + name: ECOSYSTEM, + package_manager: package_manager, + language: language + ), + T.nilable(Ecosystem) + ) + end + private sig { returns(T::Array[Dependabot::Dependency]) } @@ -37,6 +51,20 @@ def dependencies end, T.nilable(T::Array[Dependabot::Dependency])) end + sig { returns(T.nilable(Dependabot::Nuget::NativeDiscoveryJsonReader)) } + def content + @content ||= T.let(begin + directory = source&.directory || "/" + discovery_json_reader = NativeDiscoveryJsonReader.run_discovery_in_directory( + repo_contents_path: T.must(repo_contents_path), + directory: directory, + credentials: credentials + ) + + discovery_json_reader + end, T.nilable(Dependabot::Nuget::NativeDiscoveryJsonReader)) + end + sig { override.void } def check_required_files requirement_files = dependencies.flat_map do |dep| @@ -53,6 +81,74 @@ def check_required_files "No project file." ) end + + sig { returns(T.nilable(Ecosystem::VersionManager)) } + def language + # Historically new version of language is released with incremental update of + # .Net version, so we tie the language with framework version for metric collection + + nomenclature = "#{language_type}#{framework_version&.first}".strip + + Dependabot.logger.info("Detected language and framework #{nomenclature}") + + case language_type + + when CSharpLanguage::TYPE + CSharpLanguage.new(nomenclature) + + when VBLanguage::TYPE + VBLanguage.new(nomenclature) + + when FSharpLanguage::TYPE + FSharpLanguage.new(nomenclature) + + when DotNet::TYPE + DotNet.new(nomenclature) + + end + end + + sig { returns(T.nilable(T::Array[T.nilable(String)])) } + def framework_version + workplace_json = T.let(content.send(:workspace_discovery), + T.nilable(Dependabot::Nuget::NativeWorkspaceDiscovery)) + project_json = T.let(workplace_json.send(:projects), + T::Array[NativeProjectDiscovery]) + project_json.map do |framework| + version = T.let(T.let(framework.instance_variable_get(:@target_frameworks), T::Array[String]).first, + T.nilable(String)) + + "-" + version if version + end + rescue StandardError + nil + end + + sig { returns(T.nilable(String)) } + def language_type + requirement_files = dependencies.flat_map do |dep| + dep.requirements.map { |r| T.let(r.fetch(:file), String) } + end.uniq + + return "cs" if requirement_files.any? { |f| File.basename(f).match?(/\.csproj$/) } + return "vb" if requirement_files.any? { |f| File.basename(f).match?(/\.vbproj$/) } + return "fs" if requirement_files.any? { |f| File.basename(f).match?(/\.fsproj$/) } + + # return a fallback to avoid falling to exception + "dotnet" + end + + sig { returns(Ecosystem::VersionManager) } + def package_manager + NugetPackageManager.new(nuget_version) + end + + sig { returns(T.nilable(String)) } + def nuget_version + SharedHelpers.run_shell_command("dotnet nuget --version").split("Command Line").last&.strip + rescue StandardError + nil + end end end end diff --git a/nuget/lib/dependabot/nuget/language.rb b/nuget/lib/dependabot/nuget/language.rb new file mode 100644 index 0000000000..2fe6841d83 --- /dev/null +++ b/nuget/lib/dependabot/nuget/language.rb @@ -0,0 +1,82 @@ +# typed: strong +# frozen_string_literal: true + +require "sorbet-runtime" +require "dependabot/nuget/version" +require "dependabot/ecosystem" + +module Dependabot + module Nuget + class Language < Dependabot::Ecosystem::VersionManager + extend T::Sig + + sig { params(language: String, raw_version: String, requirement: T.nilable(Requirement)).void } + def initialize(language, raw_version, requirement = nil) + super(language, Version.new(raw_version), [], [], requirement) + end + end + + class CSharpLanguage < Dependabot::Ecosystem::VersionManager + extend T::Sig + + LANGUAGE = "CSharp" + TYPE = "cs" + + SUPPORTED_VERSIONS = T.let([].freeze, T::Array[Dependabot::Version]) + + DEPRECATED_VERSIONS = T.let([].freeze, T::Array[Dependabot::Version]) + + sig { params(language: String, requirement: T.nilable(Requirement)).void } + def initialize(language, requirement = nil) + super(language, Version.new(nil), [], [], requirement) + end + end + + class VBLanguage < Dependabot::Ecosystem::VersionManager + extend T::Sig + + LANGUAGE = "VB" + TYPE = "vb" + + SUPPORTED_VERSIONS = T.let([].freeze, T::Array[Dependabot::Version]) + + DEPRECATED_VERSIONS = T.let([].freeze, T::Array[Dependabot::Version]) + + sig { params(language: String, requirement: T.nilable(Requirement)).void } + def initialize(language, requirement = nil) + super(language, Version.new(nil), [], [], requirement) + end + end + + class FSharpLanguage < Dependabot::Ecosystem::VersionManager + extend T::Sig + + LANGUAGE = "FSharp" + TYPE = "fs" + + SUPPORTED_VERSIONS = T.let([].freeze, T::Array[Dependabot::Version]) + + DEPRECATED_VERSIONS = T.let([].freeze, T::Array[Dependabot::Version]) + + sig { params(language: String, requirement: T.nilable(Requirement)).void } + def initialize(language, requirement = nil) + super(language, Version.new(nil), [], [], requirement) + end + end + + class DotNet < Dependabot::Ecosystem::VersionManager + extend T::Sig + + TYPE = "dotnet" + + SUPPORTED_VERSIONS = T.let([].freeze, T::Array[Dependabot::Version]) + + DEPRECATED_VERSIONS = T.let([].freeze, T::Array[Dependabot::Version]) + + sig { params(language: String, requirement: T.nilable(Requirement)).void } + def initialize(language, requirement = nil) + super(language, Version.new(nil), [], [], requirement) + end + end + end +end diff --git a/nuget/lib/dependabot/nuget/package_manager.rb b/nuget/lib/dependabot/nuget/package_manager.rb new file mode 100644 index 0000000000..d38bc6ddd2 --- /dev/null +++ b/nuget/lib/dependabot/nuget/package_manager.rb @@ -0,0 +1,51 @@ +# typed: strong +# frozen_string_literal: true + +require "sorbet-runtime" +require "dependabot/nuget/version" +require "dependabot/ecosystem" +require "dependabot/nuget/requirement" + +module Dependabot + module Nuget + ECOSYSTEM = "dotnet" + + SUPPORTED_DART_VERSIONS = T.let([].freeze, T::Array[Dependabot::Version]) + + DEPRECATED_DART_VERSIONS = T.let([].freeze, T::Array[Dependabot::Version]) + + class NugetPackageManager < Dependabot::Ecosystem::VersionManager + extend T::Sig + + NAME = "Nuget" + + SUPPORTED_VERSIONS = T.let([].freeze, T::Array[Dependabot::Version]) + + DEPRECATED_VERSIONS = T.let([].freeze, T::Array[Dependabot::Version]) + + sig do + params( + raw_version: T.nilable(String) + ).void + end + def initialize(raw_version) + super( + NAME, + Version.new(raw_version), + SUPPORTED_VERSIONS, + DEPRECATED_VERSIONS + ) + end + + sig { override.returns(T::Boolean) } + def deprecated? + false + end + + sig { override.returns(T::Boolean) } + def unsupported? + false + end + end + end +end From 8c58aaa11e4ee76e29c2130ca4ec2ac71cc64a6e Mon Sep 17 00:00:00 2001 From: sachin-sandhu Date: Tue, 10 Dec 2024 10:50:55 -0500 Subject: [PATCH 02/11] attempt to fix sorbet warning --- nuget/lib/dependabot/nuget/file_parser.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/nuget/lib/dependabot/nuget/file_parser.rb b/nuget/lib/dependabot/nuget/file_parser.rb index ffdc37a0da..833f0d8889 100644 --- a/nuget/lib/dependabot/nuget/file_parser.rb +++ b/nuget/lib/dependabot/nuget/file_parser.rb @@ -7,6 +7,7 @@ require "dependabot/nuget/native_discovery/native_discovery_json_reader" require "dependabot/nuget/native_helpers" require "dependabot/nuget/package_manager" +require "dependabot/nuget/native_discovery/native_workspace_discovery" require "dependabot/nuget/language" require "sorbet-runtime" From 2e66d8d5c1c83d994fd09f1cd9a4bc53465bfbdd Mon Sep 17 00:00:00 2001 From: sachin-sandhu Date: Tue, 10 Dec 2024 11:26:06 -0500 Subject: [PATCH 03/11] adds unit tests --- nuget/lib/dependabot/nuget/package_manager.rb | 2 +- .../nuget/nuget_package_manager_spec.rb | 33 +++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 nuget/spec/dependabot/nuget/nuget_package_manager_spec.rb diff --git a/nuget/lib/dependabot/nuget/package_manager.rb b/nuget/lib/dependabot/nuget/package_manager.rb index d38bc6ddd2..a38abad14a 100644 --- a/nuget/lib/dependabot/nuget/package_manager.rb +++ b/nuget/lib/dependabot/nuget/package_manager.rb @@ -17,7 +17,7 @@ module Nuget class NugetPackageManager < Dependabot::Ecosystem::VersionManager extend T::Sig - NAME = "Nuget" + NAME = "nuget" SUPPORTED_VERSIONS = T.let([].freeze, T::Array[Dependabot::Version]) diff --git a/nuget/spec/dependabot/nuget/nuget_package_manager_spec.rb b/nuget/spec/dependabot/nuget/nuget_package_manager_spec.rb new file mode 100644 index 0000000000..533ddea83a --- /dev/null +++ b/nuget/spec/dependabot/nuget/nuget_package_manager_spec.rb @@ -0,0 +1,33 @@ +# typed: false +# frozen_string_literal: true + +require "dependabot/nuget/package_manager" +require "dependabot/ecosystem" +require "spec_helper" + +RSpec.describe Dependabot::Nuget::NugetPackageManager do + let(:package_manager) { described_class.new("6.5.0") } + + describe "#initialize" do + context "when version is a String" do + it "sets the version correctly" do + expect(package_manager.version).to eq("6.5.0") + end + + it "sets the name correctly" do + expect(package_manager.name).to eq("nuget") + end + end + + context "when nuget version extracted is well formed" do + # If this test starts failing, you need to adjust the "nuget_version" function + # to return a valid version in format x.x, x.x.x etc. examples: 3.12.5, 3.12 along with + # following block + version = Dependabot::SharedHelpers.run_shell_command("dotnet nuget --version").split("Command Line").last&.strip + + it "does not raise error" do + expect(version.match(/^\d+(?:\.\d+)*$/)).to be_truthy + end + end + end +end From 5c9c1f36ef57d57ceca571212f6870dfcbb2d018 Mon Sep 17 00:00:00 2001 From: sachin-sandhu Date: Tue, 10 Dec 2024 20:41:19 -0500 Subject: [PATCH 04/11] more fixes --- nuget/lib/dependabot/nuget/package_manager.rb | 4 +- .../spec/dependabot/nuget/file_parser_spec.rb | 155 ++++++++++++++++++ nuget/spec/dependabot/nuget/language_spec.rb | 38 +++++ .../nuget/nuget_package_manager_spec.rb | 12 ++ 4 files changed, 207 insertions(+), 2 deletions(-) create mode 100644 nuget/spec/dependabot/nuget/language_spec.rb diff --git a/nuget/lib/dependabot/nuget/package_manager.rb b/nuget/lib/dependabot/nuget/package_manager.rb index a38abad14a..1966e1d96c 100644 --- a/nuget/lib/dependabot/nuget/package_manager.rb +++ b/nuget/lib/dependabot/nuget/package_manager.rb @@ -10,9 +10,9 @@ module Dependabot module Nuget ECOSYSTEM = "dotnet" - SUPPORTED_DART_VERSIONS = T.let([].freeze, T::Array[Dependabot::Version]) + SUPPORTED_VERSIONS = T.let([].freeze, T::Array[Dependabot::Version]) - DEPRECATED_DART_VERSIONS = T.let([].freeze, T::Array[Dependabot::Version]) + DEPRECATED_VERSIONS = T.let([].freeze, T::Array[Dependabot::Version]) class NugetPackageManager < Dependabot::Ecosystem::VersionManager extend T::Sig diff --git a/nuget/spec/dependabot/nuget/file_parser_spec.rb b/nuget/spec/dependabot/nuget/file_parser_spec.rb index d6d0d0aecc..9981c770fa 100644 --- a/nuget/spec/dependabot/nuget/file_parser_spec.rb +++ b/nuget/spec/dependabot/nuget/file_parser_spec.rb @@ -985,8 +985,163 @@ def intercept_native_tools(discovery_content_hash:) it "returns the correct dependency set" do run_parser_test do |parser| dependencies = parser.parse + ecosystem = parser.ecosystem + package_manager = ecosystem.package_manager + language = ecosystem.language + + expect(dependencies.length).to eq(1) + expect(dependencies[0].name).to eq("Package.E") + + expect(ecosystem.name).to eq "dotnet" + expect(package_manager.requirement).to be_nil + expect(language.name).to eq "cs-net8.0" + expect(language.requirement).to be_nil + end + end + end + + context "when a vb proj file with a concrete targeted framework is set" do + let(:vbproj_file) do + Dependabot::DependencyFile.new( + name: "my.vbproj", + content: + <<~XML + + + dotnet472 + + + + + + XML + ) + end + + before do + intercept_native_tools( + discovery_content_hash: { + Path: "", + IsSuccess: true, + Projects: [{ + FilePath: "my.vbproj", + Dependencies: [{ + Name: "Package.E", + Version: "1.2.3", # regular version _is_ reported + Type: "PackageReference", + EvaluationResult: nil, + TargetFrameworks: ["dotnet472"], + IsDevDependency: false, + IsDirect: false, + IsTransitive: false, + IsOverride: false, + IsUpdate: false, + InfoUrl: nil + }], + IsSuccess: true, + Properties: [{ + Name: "TargetFramework", + Value: "dotnet472", + SourceFilePath: "my.vbproj" + }], + TargetFrameworks: ["dotnet472"], + ReferencedProjectPaths: [], + ImportedFiles: [], + AdditionalFiles: [] + }], + GlobalJson: nil, + DotNetToolsJson: nil + } + ) + end + + it "returns the correct ecosystem and language nomenclature set" do + run_parser_test do |parser| + dependencies = parser.parse + ecosystem = parser.ecosystem + package_manager = ecosystem.package_manager + language = ecosystem.language + expect(dependencies.length).to eq(1) expect(dependencies[0].name).to eq("Package.E") + + expect(ecosystem.name).to eq "dotnet" + expect(package_manager.requirement).to be_nil + expect(language.name).to eq "vb-dotnet472" + expect(language.requirement).to be_nil + end + end + end + + context "when a fs proj file with a concrete targeted framework is set" do + let(:vbproj_file) do + Dependabot::DependencyFile.new( + name: "my.fsproj", + content: + <<~XML + + + dotnet472 + + + + + + XML + ) + end + + before do + intercept_native_tools( + discovery_content_hash: { + Path: "", + IsSuccess: true, + Projects: [{ + FilePath: "my.fsproj", + Dependencies: [{ + Name: "Package.E", + Version: "1.2.3", # regular version _is_ reported + Type: "PackageReference", + EvaluationResult: nil, + TargetFrameworks: [""], + IsDevDependency: false, + IsDirect: false, + IsTransitive: false, + IsOverride: false, + IsUpdate: false, + InfoUrl: nil + }], + IsSuccess: true, + Properties: [{ + Name: "TargetFramework", + Value: "", + SourceFilePath: "my.fsproj" + }], + TargetFrameworks: [""], + ReferencedProjectPaths: [], + ImportedFiles: [], + AdditionalFiles: [] + }], + GlobalJson: nil, + DotNetToolsJson: nil + } + ) + end + + it "returns the correct ecosystem and language nomenclature set" do + run_parser_test do |parser| + dependencies = parser.parse + ecosystem = parser.ecosystem + package_manager = ecosystem.package_manager + language = ecosystem.language + + expect(dependencies.length).to eq(1) + expect(dependencies[0].name).to eq("Package.E") + + expect(ecosystem.name).to eq "dotnet" + expect(package_manager.requirement).to be_nil + expect(language.name).to eq "fs-" + expect(language.requirement).to be_nil end end end diff --git a/nuget/spec/dependabot/nuget/language_spec.rb b/nuget/spec/dependabot/nuget/language_spec.rb new file mode 100644 index 0000000000..e358819a03 --- /dev/null +++ b/nuget/spec/dependabot/nuget/language_spec.rb @@ -0,0 +1,38 @@ +# typed: false +# frozen_string_literal: true + +require "dependabot/nuget/language" +require "dependabot/nuget/requirement" +require "dependabot/ecosystem" +require "spec_helper" + +RSpec.describe Dependabot::Nuget::Language do + describe "#initialize" do + context "when version and requirement are both strings initially" do + let(:language) { Dependabot::Nuget::CSharpLanguage.new(name) } + let(:name) { "cs-dotnet472" } + + it "sets the name correctly" do + expect(language.name).to eq("cs-dotnet472") + end + end + + context "when version and requirement are both strings initially" do + let(:language) { Dependabot::Nuget::VBLanguage.new(name) } + let(:name) { "vb-net35" } + + it "sets the name correctly" do + expect(language.name).to eq("vb-net35") + end + end + + context "when version and requirement are both strings initially" do + let(:language) { Dependabot::Nuget::FSharpLanguage.new(name) } + let(:name) { "fs-netstandard1.5" } + + it "sets the name correctly" do + expect(language.name).to eq("fs-netstandard1.5") + end + end + end +end diff --git a/nuget/spec/dependabot/nuget/nuget_package_manager_spec.rb b/nuget/spec/dependabot/nuget/nuget_package_manager_spec.rb index 533ddea83a..901eccb6d0 100644 --- a/nuget/spec/dependabot/nuget/nuget_package_manager_spec.rb +++ b/nuget/spec/dependabot/nuget/nuget_package_manager_spec.rb @@ -19,6 +19,18 @@ end end + describe "#deprecated_versions" do + it "returns deprecated versions" do + expect(package_manager.deprecated_versions).to eq(Dependabot::Nuget::NugetPackageManager::DEPRECATED_VERSIONS) + end + end + + describe "#supported_versions" do + it "returns supported versions" do + expect(package_manager.supported_versions).to eq(Dependabot::Nuget::NugetPackageManager::SUPPORTED_VERSIONS) + end + end + context "when nuget version extracted is well formed" do # If this test starts failing, you need to adjust the "nuget_version" function # to return a valid version in format x.x, x.x.x etc. examples: 3.12.5, 3.12 along with From c5949abd7944ba1c47ebb99a2d2f55db74b35806 Mon Sep 17 00:00:00 2001 From: sachin-sandhu Date: Wed, 11 Dec 2024 00:33:36 -0500 Subject: [PATCH 05/11] more fixes --- nuget/lib/dependabot/nuget/file_parser.rb | 11 +++++------ nuget/spec/dependabot/nuget/file_parser_spec.rb | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/nuget/lib/dependabot/nuget/file_parser.rb b/nuget/lib/dependabot/nuget/file_parser.rb index 833f0d8889..49dde6b00b 100644 --- a/nuget/lib/dependabot/nuget/file_parser.rb +++ b/nuget/lib/dependabot/nuget/file_parser.rb @@ -7,7 +7,8 @@ require "dependabot/nuget/native_discovery/native_discovery_json_reader" require "dependabot/nuget/native_helpers" require "dependabot/nuget/package_manager" -require "dependabot/nuget/native_discovery/native_workspace_discovery" +require "dependabot/nuget/native_discovery/native_dependency_file_discovery" +require "dependabot/nuget/native_discovery/native_project_discovery" require "dependabot/nuget/language" require "sorbet-runtime" @@ -88,7 +89,7 @@ def language # Historically new version of language is released with incremental update of # .Net version, so we tie the language with framework version for metric collection - nomenclature = "#{language_type}#{framework_version&.first}".strip + nomenclature = "#{language_type} #{framework_version&.first}".strip.tr(" ", "-") Dependabot.logger.info("Detected language and framework #{nomenclature}") @@ -116,10 +117,8 @@ def framework_version project_json = T.let(workplace_json.send(:projects), T::Array[NativeProjectDiscovery]) project_json.map do |framework| - version = T.let(T.let(framework.instance_variable_get(:@target_frameworks), T::Array[String]).first, - T.nilable(String)) - - "-" + version if version + T.let(T.let(framework.instance_variable_get(:@target_frameworks), T::Array[String]).first, + T.nilable(String)) end rescue StandardError nil diff --git a/nuget/spec/dependabot/nuget/file_parser_spec.rb b/nuget/spec/dependabot/nuget/file_parser_spec.rb index 9981c770fa..920393becf 100644 --- a/nuget/spec/dependabot/nuget/file_parser_spec.rb +++ b/nuget/spec/dependabot/nuget/file_parser_spec.rb @@ -1140,7 +1140,7 @@ def intercept_native_tools(discovery_content_hash:) expect(ecosystem.name).to eq "dotnet" expect(package_manager.requirement).to be_nil - expect(language.name).to eq "fs-" + expect(language.name).to eq "fs" expect(language.requirement).to be_nil end end From 817438fc11161bd9b65d9285acf8eb742819ba4f Mon Sep 17 00:00:00 2001 From: sachin-sandhu Date: Wed, 11 Dec 2024 01:07:54 -0500 Subject: [PATCH 06/11] more fixes --- nuget/lib/dependabot/nuget/file_parser.rb | 25 ++++++++++++++--------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/nuget/lib/dependabot/nuget/file_parser.rb b/nuget/lib/dependabot/nuget/file_parser.rb index 49dde6b00b..fb1d0a1985 100644 --- a/nuget/lib/dependabot/nuget/file_parser.rb +++ b/nuget/lib/dependabot/nuget/file_parser.rb @@ -53,7 +53,7 @@ def dependencies end, T.nilable(T::Array[Dependabot::Dependency])) end - sig { returns(T.nilable(Dependabot::Nuget::NativeDiscoveryJsonReader)) } + sig { returns(T.nilable(T::Array[Dependabot::Nuget::NativeProjectDiscovery])) } def content @content ||= T.let(begin directory = source&.directory || "/" @@ -63,8 +63,8 @@ def content credentials: credentials ) - discovery_json_reader - end, T.nilable(Dependabot::Nuget::NativeDiscoveryJsonReader)) + discovery_json_reader.workspace_discovery&.projects + end, T.nilable(T::Array[Dependabot::Nuget::NativeProjectDiscovery])) end sig { override.void } @@ -112,13 +112,18 @@ def language sig { returns(T.nilable(T::Array[T.nilable(String)])) } def framework_version - workplace_json = T.let(content.send(:workspace_discovery), - T.nilable(Dependabot::Nuget::NativeWorkspaceDiscovery)) - project_json = T.let(workplace_json.send(:projects), - T::Array[NativeProjectDiscovery]) - project_json.map do |framework| - T.let(T.let(framework.instance_variable_get(:@target_frameworks), T::Array[String]).first, - T.nilable(String)) + # x debugger + + # T.nilable(Dependabot::Nuget::NativeWorkspaceDiscovery)) + # workplace_json = T.let(content.send(:workspace_discovery), + # T.nilable(Dependabot::Nuget::NativeWorkspaceDiscovery)) + # project_json = T.let(workplace_json.send(:projects), + # T::Array[NativeProjectDiscovery]) + content&.map do |framework| + # T.let(T.let(framework.instance_variable_get(:@target_frameworks), T::Array[String]).first, + # T.nilable(String)) + # debugger + T.let(framework.instance_variable_get(:@target_frameworks), T::Array[String]).first end rescue StandardError nil From 1727bc71d9e3fb1e4ea8b5ff87e7600ecdb90085 Mon Sep 17 00:00:00 2001 From: sachin-sandhu Date: Wed, 11 Dec 2024 01:16:49 -0500 Subject: [PATCH 07/11] more fixes --- .github/workflows/sorbet.yml | 4 +++ nuget/lib/dependabot/nuget/file_parser.rb | 37 +++++++---------------- 2 files changed, 15 insertions(+), 26 deletions(-) diff --git a/.github/workflows/sorbet.yml b/.github/workflows/sorbet.yml index c2073a5777..d089e1f01f 100644 --- a/.github/workflows/sorbet.yml +++ b/.github/workflows/sorbet.yml @@ -20,6 +20,10 @@ jobs: with: bundler-cache: true + - run: bundle exec srb --version + + - run: ruby -v + - run: bundle exec tapioca gem --verify - run: bundle exec srb tc diff --git a/nuget/lib/dependabot/nuget/file_parser.rb b/nuget/lib/dependabot/nuget/file_parser.rb index fb1d0a1985..c68b06376d 100644 --- a/nuget/lib/dependabot/nuget/file_parser.rb +++ b/nuget/lib/dependabot/nuget/file_parser.rb @@ -1,5 +1,4 @@ -# typed: strong -# frozen_string_literal: true +# typed: strict require "dependabot/dependency" require "dependabot/file_parsers" @@ -7,8 +6,6 @@ require "dependabot/nuget/native_discovery/native_discovery_json_reader" require "dependabot/nuget/native_helpers" require "dependabot/nuget/package_manager" -require "dependabot/nuget/native_discovery/native_dependency_file_discovery" -require "dependabot/nuget/native_discovery/native_project_discovery" require "dependabot/nuget/language" require "sorbet-runtime" @@ -41,10 +38,10 @@ def ecosystem private sig { returns(T::Array[Dependabot::Dependency]) } - def dependencies - @dependencies ||= T.let(begin + def content_json + @content_json ||= T.let(begin directory = source&.directory || "/" - discovery_json_reader = NativeDiscoveryJsonReader.run_discovery_in_directory( + discovery_json_reader = Dependabot::Nuget::NativeDiscoveryJsonReader.run_discovery_in_directory( repo_contents_path: T.must(repo_contents_path), directory: directory, credentials: credentials @@ -53,18 +50,17 @@ def dependencies end, T.nilable(T::Array[Dependabot::Dependency])) end - sig { returns(T.nilable(T::Array[Dependabot::Nuget::NativeProjectDiscovery])) } - def content - @content ||= T.let(begin + sig { returns(T::Array[Dependabot::Dependency]) } + def dependencies + @dependencies ||= T.let(begin directory = source&.directory || "/" discovery_json_reader = NativeDiscoveryJsonReader.run_discovery_in_directory( repo_contents_path: T.must(repo_contents_path), directory: directory, credentials: credentials ) - - discovery_json_reader.workspace_discovery&.projects - end, T.nilable(T::Array[Dependabot::Nuget::NativeProjectDiscovery])) + discovery_json_reader.dependency_set.dependencies + end, T.nilable(T::Array[Dependabot::Dependency])) end sig { override.void } @@ -112,19 +108,8 @@ def language sig { returns(T.nilable(T::Array[T.nilable(String)])) } def framework_version - # x debugger - - # T.nilable(Dependabot::Nuget::NativeWorkspaceDiscovery)) - # workplace_json = T.let(content.send(:workspace_discovery), - # T.nilable(Dependabot::Nuget::NativeWorkspaceDiscovery)) - # project_json = T.let(workplace_json.send(:projects), - # T::Array[NativeProjectDiscovery]) - content&.map do |framework| - # T.let(T.let(framework.instance_variable_get(:@target_frameworks), T::Array[String]).first, - # T.nilable(String)) - # debugger - T.let(framework.instance_variable_get(:@target_frameworks), T::Array[String]).first - end + # pending due to some issue with sorbet during CI tests + [] rescue StandardError nil end From e753d18cbe89ba94c4a25140b41175e72569a2de Mon Sep 17 00:00:00 2001 From: sachin-sandhu Date: Wed, 11 Dec 2024 10:11:27 -0500 Subject: [PATCH 08/11] change module --- nuget/lib/dependabot/nuget/file_fetcher.rb | 2 +- nuget/lib/dependabot/nuget/file_parser.rb | 2 +- nuget/lib/dependabot/nuget/file_updater.rb | 10 +- .../native_discovery_json_reader.rb | 420 +++++++++--------- .../native_update_checker.rb | 6 +- 5 files changed, 222 insertions(+), 218 deletions(-) diff --git a/nuget/lib/dependabot/nuget/file_fetcher.rb b/nuget/lib/dependabot/nuget/file_fetcher.rb index 5dacfd983e..3ad640b7b2 100644 --- a/nuget/lib/dependabot/nuget/file_fetcher.rb +++ b/nuget/lib/dependabot/nuget/file_fetcher.rb @@ -26,7 +26,7 @@ def self.required_files_message sig { override.returns(T::Array[DependencyFile]) } def fetch_files - discovery_json_reader = NativeDiscoveryJsonReader.run_discovery_in_directory( + discovery_json_reader = Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.run_discovery_in_directory( repo_contents_path: T.must(repo_contents_path), directory: directory, credentials: credentials diff --git a/nuget/lib/dependabot/nuget/file_parser.rb b/nuget/lib/dependabot/nuget/file_parser.rb index c68b06376d..9b5ff44a26 100644 --- a/nuget/lib/dependabot/nuget/file_parser.rb +++ b/nuget/lib/dependabot/nuget/file_parser.rb @@ -41,7 +41,7 @@ def ecosystem def content_json @content_json ||= T.let(begin directory = source&.directory || "/" - discovery_json_reader = Dependabot::Nuget::NativeDiscoveryJsonReader.run_discovery_in_directory( + discovery_json_reader = Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.run_discovery_in_directory( repo_contents_path: T.must(repo_contents_path), directory: directory, credentials: credentials diff --git a/nuget/lib/dependabot/nuget/file_updater.rb b/nuget/lib/dependabot/nuget/file_updater.rb index 39393b9ae5..0c95e791d8 100644 --- a/nuget/lib/dependabot/nuget/file_updater.rb +++ b/nuget/lib/dependabot/nuget/file_updater.rb @@ -57,7 +57,7 @@ def updated_dependency_files try_update_projects(dependency) || try_update_json(dependency) end updated_files = dependency_files.filter_map do |f| - dependency_file_path = NativeDiscoveryJsonReader.dependency_file_path( + dependency_file_path = Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.dependency_file_path( repo_contents_path: T.must(repo_contents_path), dependency_file: f ) @@ -97,7 +97,7 @@ def try_update_projects(dependency) # run update for each project file project_files.each do |project_file| project_dependencies = project_dependencies(project_file) - dependency_file_path = NativeDiscoveryJsonReader.dependency_file_path( + dependency_file_path = Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.dependency_file_path( repo_contents_path: T.must(repo_contents_path), dependency_file: project_file ) @@ -128,7 +128,7 @@ def try_update_json(dependency) # We just need to feed the updater a project file, grab the first project_file = T.must(project_files.first) - dependency_file_path = NativeDiscoveryJsonReader.dependency_file_path( + dependency_file_path = Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.dependency_file_path( repo_contents_path: T.must(repo_contents_path), dependency_file: project_file ) @@ -171,10 +171,10 @@ def testonly_update_tooling_calls sig { returns(T.nilable(NativeWorkspaceDiscovery)) } def workspace dependency_file_paths = dependency_files.map do |f| - NativeDiscoveryJsonReader.dependency_file_path(repo_contents_path: T.must(repo_contents_path), + Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.dependency_file_path(repo_contents_path: T.must(repo_contents_path), dependency_file: f) end - NativeDiscoveryJsonReader.load_discovery_for_dependency_file_paths(dependency_file_paths).workspace_discovery + Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.load_discovery_for_dependency_file_paths(dependency_file_paths).workspace_discovery end sig { params(project_file: Dependabot::DependencyFile).returns(T::Array[String]) } diff --git a/nuget/lib/dependabot/nuget/native_discovery/native_discovery_json_reader.rb b/nuget/lib/dependabot/nuget/native_discovery/native_discovery_json_reader.rb index 7bcaadb865..cad93b4d3a 100644 --- a/nuget/lib/dependabot/nuget/native_discovery/native_discovery_json_reader.rb +++ b/nuget/lib/dependabot/nuget/native_discovery/native_discovery_json_reader.rb @@ -10,256 +10,260 @@ module Dependabot module Nuget - class NativeDiscoveryJsonReader - extend T::Sig + module NativeDiscovery + class NativeDiscoveryJsonReader + extend T::Sig - sig { returns(T::Hash[String, NativeDiscoveryJsonReader]) } - def self.cache_directory_to_discovery_json_reader - CacheManager.cache("cache_directory_to_discovery_json_reader") - end - - sig { returns(T::Hash[String, NativeDiscoveryJsonReader]) } - def self.cache_dependency_file_paths_to_discovery_json_reader - CacheManager.cache("cache_dependency_file_paths_to_discovery_json_reader") - end - - sig { returns(T::Hash[String, String]) } - def self.cache_dependency_file_paths_to_discovery_json_path - CacheManager.cache("cache_dependency_file_paths_to_discovery_json_path") - end - - sig { void } - def self.testonly_clear_caches - cache_directory_to_discovery_json_reader.clear - cache_dependency_file_paths_to_discovery_json_reader.clear - cache_dependency_file_paths_to_discovery_json_path.clear - end + sig { returns(T::Hash[String, NativeDiscoveryJsonReader]) } + def self.cache_directory_to_discovery_json_reader + CacheManager.cache("cache_directory_to_discovery_json_reader") + end - sig { void } - def self.testonly_clear_discovery_files - # this will get recreated when necessary - FileUtils.rm_rf(discovery_directory) - end + sig { returns(T::Hash[String, NativeDiscoveryJsonReader]) } + def self.cache_dependency_file_paths_to_discovery_json_reader + CacheManager.cache("cache_dependency_file_paths_to_discovery_json_reader") + end - # Runs NuGet dependency discovery in the given directory and returns a new instance of NativeDiscoveryJsonReader. - # The location of the resultant JSON file is saved. - sig do - params( - repo_contents_path: String, - directory: String, - credentials: T::Array[Dependabot::Credential] - ).returns(NativeDiscoveryJsonReader) - end - def self.run_discovery_in_directory(repo_contents_path:, directory:, credentials:) - # run discovery - job_file_path = ENV.fetch("DEPENDABOT_JOB_PATH") - discovery_json_path = discovery_file_path_from_workspace_path(directory) - unless File.exist?(discovery_json_path) - NativeHelpers.run_nuget_discover_tool(job_path: job_file_path, - repo_root: repo_contents_path, - workspace_path: directory, - output_path: discovery_json_path, - credentials: credentials) - - Dependabot.logger.info("Discovery JSON content: #{File.read(discovery_json_path)}") + sig { returns(T::Hash[String, String]) } + def self.cache_dependency_file_paths_to_discovery_json_path + CacheManager.cache("cache_dependency_file_paths_to_discovery_json_path") end - load_discovery_for_directory(repo_contents_path: repo_contents_path, directory: directory) - end - # Loads NuGet dependency discovery for the given directory and returns a new instance of - # NativeDiscoveryJsonReader and caches the resultant object. - sig { params(repo_contents_path: String, directory: String).returns(NativeDiscoveryJsonReader) } - def self.load_discovery_for_directory(repo_contents_path:, directory:) - cache_directory_to_discovery_json_reader[directory] ||= begin - discovery_json_reader = discovery_json_reader(repo_contents_path: repo_contents_path, - workspace_path: directory) - cache_directory_to_discovery_json_reader[directory] = discovery_json_reader - dependency_file_cache_key = cache_key_from_dependency_file_paths(discovery_json_reader.dependency_file_paths) - cache_dependency_file_paths_to_discovery_json_reader[dependency_file_cache_key] = discovery_json_reader - discovery_file_path = discovery_file_path_from_workspace_path(directory) - cache_dependency_file_paths_to_discovery_json_path[dependency_file_cache_key] = discovery_file_path - - discovery_json_reader + sig { void } + def self.testonly_clear_caches + cache_directory_to_discovery_json_reader.clear + cache_dependency_file_paths_to_discovery_json_reader.clear + cache_dependency_file_paths_to_discovery_json_path.clear end - end - # Retrieves the cached NativeDiscoveryJsonReader object for the given dependency file paths. - sig { params(dependency_file_paths: T::Array[String]).returns(NativeDiscoveryJsonReader) } - def self.load_discovery_for_dependency_file_paths(dependency_file_paths) - dependency_file_cache_key = cache_key_from_dependency_file_paths(dependency_file_paths) - T.must(cache_dependency_file_paths_to_discovery_json_reader[dependency_file_cache_key]) - end + sig { void } + def self.testonly_clear_discovery_files + # this will get recreated when necessary + FileUtils.rm_rf(discovery_directory) + end - # Retrieves the cached location of the discovery JSON file for the given dependency file paths. - sig { params(dependency_file_paths: T::Array[String]).returns(String) } - def self.get_discovery_json_path_for_dependency_file_paths(dependency_file_paths) - dependency_file_cache_key = cache_key_from_dependency_file_paths(dependency_file_paths) - T.must(cache_dependency_file_paths_to_discovery_json_path[dependency_file_cache_key]) - end + # Runs NuGet dependency discovery in the given directory and returns a new instance of NativeDiscoveryJsonReader. + # The location of the resultant JSON file is saved. + sig do + params( + repo_contents_path: String, + directory: String, + credentials: T::Array[Dependabot::Credential] + ).returns(NativeDiscoveryJsonReader) + end + def self.run_discovery_in_directory(repo_contents_path:, directory:, credentials:) + # run discovery + job_file_path = ENV.fetch("DEPENDABOT_JOB_PATH") + discovery_json_path = discovery_file_path_from_workspace_path(directory) + unless File.exist?(discovery_json_path) + NativeHelpers.run_nuget_discover_tool(job_path: job_file_path, + repo_root: repo_contents_path, + workspace_path: directory, + output_path: discovery_json_path, + credentials: credentials) + + Dependabot.logger.info("Discovery JSON content: #{File.read(discovery_json_path)}") + end + load_discovery_for_directory(repo_contents_path: repo_contents_path, directory: directory) + end - sig { params(repo_contents_path: String, dependency_file: Dependabot::DependencyFile).returns(String) } - def self.dependency_file_path(repo_contents_path:, dependency_file:) - dep_file_path = Pathname.new(File.join(dependency_file.directory, dependency_file.name)).cleanpath.to_path - dep_file_path.delete_prefix("#{repo_contents_path}/") - end + # Loads NuGet dependency discovery for the given directory and returns a new instance of + # NativeDiscoveryJsonReader and caches the resultant object. + sig { params(repo_contents_path: String, directory: String).returns(NativeDiscoveryJsonReader) } + def self.load_discovery_for_directory(repo_contents_path:, directory:) + cache_directory_to_discovery_json_reader[directory] ||= begin + discovery_json_reader = discovery_json_reader(repo_contents_path: repo_contents_path, + workspace_path: directory) + cache_directory_to_discovery_json_reader[directory] = discovery_json_reader + dependency_file_cache_key = cache_key_from_dependency_file_paths(discovery_json_reader.dependency_file_paths) + cache_dependency_file_paths_to_discovery_json_reader[dependency_file_cache_key] = discovery_json_reader + discovery_file_path = discovery_file_path_from_workspace_path(directory) + cache_dependency_file_paths_to_discovery_json_path[dependency_file_cache_key] = discovery_file_path + + discovery_json_reader + end + end - sig { returns(String) } - def self.discovery_map_file_path - File.join(discovery_directory, "discovery_map.json") - end + # Retrieves the cached NativeDiscoveryJsonReader object for the given dependency file paths. + sig { params(dependency_file_paths: T::Array[String]).returns(NativeDiscoveryJsonReader) } + def self.load_discovery_for_dependency_file_paths(dependency_file_paths) + dependency_file_cache_key = cache_key_from_dependency_file_paths(dependency_file_paths) + T.must(cache_dependency_file_paths_to_discovery_json_reader[dependency_file_cache_key]) + end - sig { params(workspace_path: String).returns(String) } - def self.discovery_file_path_from_workspace_path(workspace_path) - # Given an update directory (also known as a workspace path), this function returns the path where the discovery - # JSON file is located. This function is called both by methods that need to write the discovery JSON file and - # by methods that need to read the discovery JSON file. This function is also called by multiple processes so - # we need a way to retain the data. This is accomplished by the following steps: - # 1. Check a well-known file for a mapping of workspace_path => discovery file path. If found, return it. - # 2. If the path is not found, generate a new path, save it to the well-known file, and return the value. - discovery_map_contents = File.exist?(discovery_map_file_path) ? File.read(discovery_map_file_path) : "{}" - discovery_map = T.let(JSON.parse(discovery_map_contents), T::Hash[String, String]) - - discovery_json_path = discovery_map[workspace_path] - if discovery_json_path - Dependabot.logger.info("Discovery JSON path for workspace path [#{workspace_path}] found in file " \ - "[#{discovery_map_file_path}] at location [#{discovery_json_path}]") - return discovery_json_path + # Retrieves the cached location of the discovery JSON file for the given dependency file paths. + sig { params(dependency_file_paths: T::Array[String]).returns(String) } + def self.get_discovery_json_path_for_dependency_file_paths(dependency_file_paths) + dependency_file_cache_key = cache_key_from_dependency_file_paths(dependency_file_paths) + T.must(cache_dependency_file_paths_to_discovery_json_path[dependency_file_cache_key]) end - # no discovery JSON path found; generate a new one, but first find a suitable location - discovery_json_counter = 1 - new_discovery_json_path = "" - loop do - new_discovery_json_path = File.join(discovery_directory, "discovery.#{discovery_json_counter}.json") - break unless File.exist?(new_discovery_json_path) + sig { params(repo_contents_path: String, dependency_file: Dependabot::DependencyFile).returns(String) } + def self.dependency_file_path(repo_contents_path:, dependency_file:) + dep_file_path = Pathname.new(File.join(dependency_file.directory, dependency_file.name)).cleanpath.to_path + dep_file_path.delete_prefix("#{repo_contents_path}/") + end - discovery_json_counter += 1 + sig { returns(String) } + def self.discovery_map_file_path + File.join(discovery_directory, "discovery_map.json") end - discovery_map[workspace_path] = new_discovery_json_path + sig { params(workspace_path: String).returns(String) } + def self.discovery_file_path_from_workspace_path(workspace_path) + # Given an update directory (also known as a workspace path), this function returns the path where the discovery + # JSON file is located. This function is called both by methods that need to write the discovery JSON file and + # by methods that need to read the discovery JSON file. This function is also called by multiple processes so + # we need a way to retain the data. This is accomplished by the following steps: + # 1. Check a well-known file for a mapping of workspace_path => discovery file path. If found, return it. + # 2. If the path is not found, generate a new path, save it to the well-known file, and return the value. + discovery_map_contents = File.exist?(discovery_map_file_path) ? File.read(discovery_map_file_path) : "{}" + discovery_map = T.let(JSON.parse(discovery_map_contents), T::Hash[String, String]) + + discovery_json_path = discovery_map[workspace_path] + if discovery_json_path + Dependabot.logger.info("Discovery JSON path for workspace path [#{workspace_path}] found in file " \ + "[#{discovery_map_file_path}] at location [#{discovery_json_path}]") + return discovery_json_path + end + + # no discovery JSON path found; generate a new one, but first find a suitable location + discovery_json_counter = 1 + new_discovery_json_path = "" + loop do + new_discovery_json_path = File.join(discovery_directory, "discovery.#{discovery_json_counter}.json") + break unless File.exist?(new_discovery_json_path) - File.write(discovery_map_file_path, discovery_map.to_json) - Dependabot.logger.info("Discovery JSON path for workspace path [#{workspace_path}] created for file " \ - "[#{discovery_map_file_path}] at location [#{new_discovery_json_path}]") - new_discovery_json_path - end + discovery_json_counter += 1 + end - sig { params(dependency_file_paths: T::Array[String]).returns(String) } - def self.cache_key_from_dependency_file_paths(dependency_file_paths) - dependency_file_paths.sort.join(",") - end + discovery_map[workspace_path] = new_discovery_json_path - sig { returns(String) } - def self.discovery_directory - t = File.join(Dir.home, ".dependabot") - FileUtils.mkdir_p(t) - t - end + File.write(discovery_map_file_path, discovery_map.to_json) + Dependabot.logger.info("Discovery JSON path for workspace path [#{workspace_path}] created for file " \ + "[#{discovery_map_file_path}] at location [#{new_discovery_json_path}]") + new_discovery_json_path + end - sig { params(repo_contents_path: String, workspace_path: String).returns(NativeDiscoveryJsonReader) } - def self.discovery_json_reader(repo_contents_path:, workspace_path:) - discovery_file_path = discovery_file_path_from_workspace_path(workspace_path) - discovery_json = DependencyFile.new( - name: Pathname.new(discovery_file_path).cleanpath.to_path, - directory: discovery_directory, - type: "file", - content: File.read(discovery_file_path) - ) - NativeDiscoveryJsonReader.new(repo_contents_path: repo_contents_path, discovery_json: discovery_json) - end + sig { params(dependency_file_paths: T::Array[String]).returns(String) } + def self.cache_key_from_dependency_file_paths(dependency_file_paths) + dependency_file_paths.sort.join(",") + end - sig { returns(T.nilable(NativeWorkspaceDiscovery)) } - attr_reader :workspace_discovery + sig { returns(String) } + def self.discovery_directory + t = File.join(Dir.home, ".dependabot") + FileUtils.mkdir_p(t) + t + end - sig { returns(Dependabot::FileParsers::Base::DependencySet) } - attr_reader :dependency_set + sig { params(repo_contents_path: String, workspace_path: String).returns(NativeDiscoveryJsonReader) } + def self.discovery_json_reader(repo_contents_path:, workspace_path:) + discovery_file_path = discovery_file_path_from_workspace_path(workspace_path) + discovery_json = DependencyFile.new( + name: Pathname.new(discovery_file_path).cleanpath.to_path, + directory: discovery_directory, + type: "file", + content: File.read(discovery_file_path) + ) + NativeDiscoveryJsonReader.new(repo_contents_path: repo_contents_path, discovery_json: discovery_json) + end - sig { returns(T::Array[String]) } - attr_reader :dependency_file_paths + sig { returns(T.nilable(NativeWorkspaceDiscovery)) } + attr_reader :workspace_discovery - sig { params(repo_contents_path: String, discovery_json: DependencyFile).void } - def initialize(repo_contents_path:, discovery_json:) - @repo_contents_path = repo_contents_path - @discovery_json = discovery_json - @workspace_discovery = T.let(read_workspace_discovery, T.nilable(Dependabot::Nuget::NativeWorkspaceDiscovery)) - @dependency_set = T.let(read_dependency_set, Dependabot::FileParsers::Base::DependencySet) - @dependency_file_paths = T.let(read_dependency_file_paths, T::Array[String]) - end + sig { returns(Dependabot::FileParsers::Base::DependencySet) } + attr_reader :dependency_set - private + sig { returns(T::Array[String]) } + attr_reader :dependency_file_paths - sig { returns(String) } - attr_reader :repo_contents_path + sig { params(repo_contents_path: String, discovery_json: DependencyFile).void } + def initialize(repo_contents_path:, discovery_json:) + @repo_contents_path = repo_contents_path + @discovery_json = discovery_json + @workspace_discovery = T.let(read_workspace_discovery, T.nilable(Dependabot::Nuget::NativeWorkspaceDiscovery)) + @dependency_set = T.let(read_dependency_set, Dependabot::FileParsers::Base::DependencySet) + @dependency_file_paths = T.let(read_dependency_file_paths, T::Array[String]) + end - sig { returns(DependencyFile) } - attr_reader :discovery_json + private - sig { returns(T.nilable(NativeWorkspaceDiscovery)) } - def read_workspace_discovery - return nil unless discovery_json.content + sig { returns(String) } + attr_reader :repo_contents_path - parsed_json = T.let(JSON.parse(T.must(discovery_json.content)), T::Hash[String, T.untyped]) - NativeWorkspaceDiscovery.from_json(parsed_json) - rescue JSON::ParserError - raise Dependabot::DependencyFileNotParseable, discovery_json.path - end + sig { returns(DependencyFile) } + attr_reader :discovery_json - sig { returns(Dependabot::FileParsers::Base::DependencySet) } - def read_dependency_set - dependency_set = Dependabot::FileParsers::Base::DependencySet.new - return dependency_set unless workspace_discovery + sig { returns(T.nilable(NativeWorkspaceDiscovery)) } + def read_workspace_discovery + return nil unless discovery_json.content - workspace_result = T.must(workspace_discovery) - workspace_result.projects.each do |project| - dependency_set += project.dependency_set + parsed_json = T.let(JSON.parse(T.must(discovery_json.content)), T::Hash[String, T.untyped]) + NativeWorkspaceDiscovery.from_json(parsed_json) + rescue JSON::ParserError + raise Dependabot::DependencyFileNotParseable, discovery_json.path end - if workspace_result.dotnet_tools_json - dependency_set += T.must(workspace_result.dotnet_tools_json).dependency_set - end - dependency_set += T.must(workspace_result.global_json).dependency_set if workspace_result.global_json - dependency_set - end + sig { returns(Dependabot::FileParsers::Base::DependencySet) } + def read_dependency_set + dependency_set = Dependabot::FileParsers::Base::DependencySet.new + return dependency_set unless workspace_discovery - sig { returns(T::Array[String]) } - def read_dependency_file_paths - dependency_file_paths = T.let([], T::Array[T.nilable(String)]) - dependency_file_paths << dependency_file_path_from_repo_path("global.json") if workspace_discovery&.global_json - if workspace_discovery&.dotnet_tools_json - dependency_file_paths << dependency_file_path_from_repo_path(".config/dotnet-tools.json") + workspace_result = T.must(workspace_discovery) + workspace_result.projects.each do |project| + dependency_set += project.dependency_set + end + if workspace_result.dotnet_tools_json + dependency_set += T.must(workspace_result.dotnet_tools_json).dependency_set + end + dependency_set += T.must(workspace_result.global_json).dependency_set if workspace_result.global_json + + dependency_set end - projects = workspace_discovery&.projects || [] - projects.each do |project| - dependency_file_paths << dependency_file_path_from_repo_path(project.file_path) - dependency_file_paths += project.imported_files.map do |f| - dependency_file_path_from_project_path(project.file_path, f) + sig { returns(T::Array[String]) } + def read_dependency_file_paths + dependency_file_paths = T.let([], T::Array[T.nilable(String)]) + if workspace_discovery&.global_json + dependency_file_paths << dependency_file_path_from_repo_path("global.json") end - dependency_file_paths += project.additional_files.map do |f| - dependency_file_path_from_project_path(project.file_path, f) + if workspace_discovery&.dotnet_tools_json + dependency_file_paths << dependency_file_path_from_repo_path(".config/dotnet-tools.json") end - end - deduped_dependency_file_paths = T.let(Set.new(dependency_file_paths.compact), T::Set[String]) - result = deduped_dependency_file_paths.sort - result - end + projects = workspace_discovery&.projects || [] + projects.each do |project| + dependency_file_paths << dependency_file_path_from_repo_path(project.file_path) + dependency_file_paths += project.imported_files.map do |f| + dependency_file_path_from_project_path(project.file_path, f) + end + dependency_file_paths += project.additional_files.map do |f| + dependency_file_path_from_project_path(project.file_path, f) + end + end - sig { params(path_parts: String).returns(T.nilable(String)) } - def dependency_file_path_from_repo_path(*path_parts) - path_parts = path_parts.map { |p| p.delete_prefix("/").delete_suffix("/") } - normalized_repo_path = Pathname.new(path_parts.join("/")).cleanpath.to_path.delete_prefix("/") - full_path = Pathname.new(File.join(repo_contents_path, normalized_repo_path)).cleanpath.to_path - return unless File.exist?(full_path) + deduped_dependency_file_paths = T.let(Set.new(dependency_file_paths.compact), T::Set[String]) + result = deduped_dependency_file_paths.sort + result + end - normalized_repo_path = "/#{normalized_repo_path}" unless normalized_repo_path.start_with?("/") - normalized_repo_path - end + sig { params(path_parts: String).returns(T.nilable(String)) } + def dependency_file_path_from_repo_path(*path_parts) + path_parts = path_parts.map { |p| p.delete_prefix("/").delete_suffix("/") } + normalized_repo_path = Pathname.new(path_parts.join("/")).cleanpath.to_path.delete_prefix("/") + full_path = Pathname.new(File.join(repo_contents_path, normalized_repo_path)).cleanpath.to_path + return unless File.exist?(full_path) - sig { params(project_path: String, relative_file_path: String).returns(T.nilable(String)) } - def dependency_file_path_from_project_path(project_path, relative_file_path) - project_directory = File.dirname(project_path) - dependency_file_path_from_repo_path(project_directory, relative_file_path) + normalized_repo_path = "/#{normalized_repo_path}" unless normalized_repo_path.start_with?("/") + normalized_repo_path + end + + sig { params(project_path: String, relative_file_path: String).returns(T.nilable(String)) } + def dependency_file_path_from_project_path(project_path, relative_file_path) + project_directory = File.dirname(project_path) + dependency_file_path_from_repo_path(project_directory, relative_file_path) + end end end end diff --git a/nuget/lib/dependabot/nuget/native_update_checker/native_update_checker.rb b/nuget/lib/dependabot/nuget/native_update_checker/native_update_checker.rb index 10fe1034c5..32ab30da5c 100644 --- a/nuget/lib/dependabot/nuget/native_update_checker/native_update_checker.rb +++ b/nuget/lib/dependabot/nuget/native_update_checker/native_update_checker.rb @@ -97,7 +97,7 @@ def dependency_file_path sig { returns(T::Array[String]) } def dependency_file_paths dependency_files.map do |file| - NativeDiscoveryJsonReader.dependency_file_path( + Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.dependency_file_path( repo_contents_path: T.must(repo_contents_path), dependency_file: file ) @@ -106,7 +106,7 @@ def dependency_file_paths sig { returns(AnalysisJsonReader) } def request_analysis - discovery_file_path = NativeDiscoveryJsonReader.get_discovery_json_path_for_dependency_file_paths( + discovery_file_path = Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.get_discovery_json_path_for_dependency_file_paths( dependency_file_paths ) analysis_folder_path = AnalysisJsonReader.temp_directory @@ -154,7 +154,7 @@ def write_dependency_info sig { returns(Dependabot::FileParsers::Base::DependencySet) } def discovered_dependencies - NativeDiscoveryJsonReader.load_discovery_for_dependency_file_paths(dependency_file_paths).dependency_set + Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.load_discovery_for_dependency_file_paths(dependency_file_paths).dependency_set end sig { override.returns(T::Boolean) } From a85dc0394a0d47c22641e9b31373233a5f5f4935 Mon Sep 17 00:00:00 2001 From: sachin-sandhu Date: Wed, 11 Dec 2024 10:28:01 -0500 Subject: [PATCH 09/11] Revert "change module" This reverts commit e753d18cbe89ba94c4a25140b41175e72569a2de. --- nuget/lib/dependabot/nuget/file_fetcher.rb | 2 +- nuget/lib/dependabot/nuget/file_parser.rb | 2 +- nuget/lib/dependabot/nuget/file_updater.rb | 10 +- .../native_discovery_json_reader.rb | 420 +++++++++--------- .../native_update_checker.rb | 6 +- 5 files changed, 218 insertions(+), 222 deletions(-) diff --git a/nuget/lib/dependabot/nuget/file_fetcher.rb b/nuget/lib/dependabot/nuget/file_fetcher.rb index 3ad640b7b2..5dacfd983e 100644 --- a/nuget/lib/dependabot/nuget/file_fetcher.rb +++ b/nuget/lib/dependabot/nuget/file_fetcher.rb @@ -26,7 +26,7 @@ def self.required_files_message sig { override.returns(T::Array[DependencyFile]) } def fetch_files - discovery_json_reader = Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.run_discovery_in_directory( + discovery_json_reader = NativeDiscoveryJsonReader.run_discovery_in_directory( repo_contents_path: T.must(repo_contents_path), directory: directory, credentials: credentials diff --git a/nuget/lib/dependabot/nuget/file_parser.rb b/nuget/lib/dependabot/nuget/file_parser.rb index 9b5ff44a26..c68b06376d 100644 --- a/nuget/lib/dependabot/nuget/file_parser.rb +++ b/nuget/lib/dependabot/nuget/file_parser.rb @@ -41,7 +41,7 @@ def ecosystem def content_json @content_json ||= T.let(begin directory = source&.directory || "/" - discovery_json_reader = Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.run_discovery_in_directory( + discovery_json_reader = Dependabot::Nuget::NativeDiscoveryJsonReader.run_discovery_in_directory( repo_contents_path: T.must(repo_contents_path), directory: directory, credentials: credentials diff --git a/nuget/lib/dependabot/nuget/file_updater.rb b/nuget/lib/dependabot/nuget/file_updater.rb index 0c95e791d8..39393b9ae5 100644 --- a/nuget/lib/dependabot/nuget/file_updater.rb +++ b/nuget/lib/dependabot/nuget/file_updater.rb @@ -57,7 +57,7 @@ def updated_dependency_files try_update_projects(dependency) || try_update_json(dependency) end updated_files = dependency_files.filter_map do |f| - dependency_file_path = Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.dependency_file_path( + dependency_file_path = NativeDiscoveryJsonReader.dependency_file_path( repo_contents_path: T.must(repo_contents_path), dependency_file: f ) @@ -97,7 +97,7 @@ def try_update_projects(dependency) # run update for each project file project_files.each do |project_file| project_dependencies = project_dependencies(project_file) - dependency_file_path = Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.dependency_file_path( + dependency_file_path = NativeDiscoveryJsonReader.dependency_file_path( repo_contents_path: T.must(repo_contents_path), dependency_file: project_file ) @@ -128,7 +128,7 @@ def try_update_json(dependency) # We just need to feed the updater a project file, grab the first project_file = T.must(project_files.first) - dependency_file_path = Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.dependency_file_path( + dependency_file_path = NativeDiscoveryJsonReader.dependency_file_path( repo_contents_path: T.must(repo_contents_path), dependency_file: project_file ) @@ -171,10 +171,10 @@ def testonly_update_tooling_calls sig { returns(T.nilable(NativeWorkspaceDiscovery)) } def workspace dependency_file_paths = dependency_files.map do |f| - Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.dependency_file_path(repo_contents_path: T.must(repo_contents_path), + NativeDiscoveryJsonReader.dependency_file_path(repo_contents_path: T.must(repo_contents_path), dependency_file: f) end - Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.load_discovery_for_dependency_file_paths(dependency_file_paths).workspace_discovery + NativeDiscoveryJsonReader.load_discovery_for_dependency_file_paths(dependency_file_paths).workspace_discovery end sig { params(project_file: Dependabot::DependencyFile).returns(T::Array[String]) } diff --git a/nuget/lib/dependabot/nuget/native_discovery/native_discovery_json_reader.rb b/nuget/lib/dependabot/nuget/native_discovery/native_discovery_json_reader.rb index cad93b4d3a..7bcaadb865 100644 --- a/nuget/lib/dependabot/nuget/native_discovery/native_discovery_json_reader.rb +++ b/nuget/lib/dependabot/nuget/native_discovery/native_discovery_json_reader.rb @@ -10,260 +10,256 @@ module Dependabot module Nuget - module NativeDiscovery - class NativeDiscoveryJsonReader - extend T::Sig + class NativeDiscoveryJsonReader + extend T::Sig - sig { returns(T::Hash[String, NativeDiscoveryJsonReader]) } - def self.cache_directory_to_discovery_json_reader - CacheManager.cache("cache_directory_to_discovery_json_reader") - end + sig { returns(T::Hash[String, NativeDiscoveryJsonReader]) } + def self.cache_directory_to_discovery_json_reader + CacheManager.cache("cache_directory_to_discovery_json_reader") + end - sig { returns(T::Hash[String, NativeDiscoveryJsonReader]) } - def self.cache_dependency_file_paths_to_discovery_json_reader - CacheManager.cache("cache_dependency_file_paths_to_discovery_json_reader") - end + sig { returns(T::Hash[String, NativeDiscoveryJsonReader]) } + def self.cache_dependency_file_paths_to_discovery_json_reader + CacheManager.cache("cache_dependency_file_paths_to_discovery_json_reader") + end - sig { returns(T::Hash[String, String]) } - def self.cache_dependency_file_paths_to_discovery_json_path - CacheManager.cache("cache_dependency_file_paths_to_discovery_json_path") - end + sig { returns(T::Hash[String, String]) } + def self.cache_dependency_file_paths_to_discovery_json_path + CacheManager.cache("cache_dependency_file_paths_to_discovery_json_path") + end - sig { void } - def self.testonly_clear_caches - cache_directory_to_discovery_json_reader.clear - cache_dependency_file_paths_to_discovery_json_reader.clear - cache_dependency_file_paths_to_discovery_json_path.clear - end + sig { void } + def self.testonly_clear_caches + cache_directory_to_discovery_json_reader.clear + cache_dependency_file_paths_to_discovery_json_reader.clear + cache_dependency_file_paths_to_discovery_json_path.clear + end - sig { void } - def self.testonly_clear_discovery_files - # this will get recreated when necessary - FileUtils.rm_rf(discovery_directory) - end + sig { void } + def self.testonly_clear_discovery_files + # this will get recreated when necessary + FileUtils.rm_rf(discovery_directory) + end - # Runs NuGet dependency discovery in the given directory and returns a new instance of NativeDiscoveryJsonReader. - # The location of the resultant JSON file is saved. - sig do - params( - repo_contents_path: String, - directory: String, - credentials: T::Array[Dependabot::Credential] - ).returns(NativeDiscoveryJsonReader) - end - def self.run_discovery_in_directory(repo_contents_path:, directory:, credentials:) - # run discovery - job_file_path = ENV.fetch("DEPENDABOT_JOB_PATH") - discovery_json_path = discovery_file_path_from_workspace_path(directory) - unless File.exist?(discovery_json_path) - NativeHelpers.run_nuget_discover_tool(job_path: job_file_path, - repo_root: repo_contents_path, - workspace_path: directory, - output_path: discovery_json_path, - credentials: credentials) - - Dependabot.logger.info("Discovery JSON content: #{File.read(discovery_json_path)}") - end - load_discovery_for_directory(repo_contents_path: repo_contents_path, directory: directory) + # Runs NuGet dependency discovery in the given directory and returns a new instance of NativeDiscoveryJsonReader. + # The location of the resultant JSON file is saved. + sig do + params( + repo_contents_path: String, + directory: String, + credentials: T::Array[Dependabot::Credential] + ).returns(NativeDiscoveryJsonReader) + end + def self.run_discovery_in_directory(repo_contents_path:, directory:, credentials:) + # run discovery + job_file_path = ENV.fetch("DEPENDABOT_JOB_PATH") + discovery_json_path = discovery_file_path_from_workspace_path(directory) + unless File.exist?(discovery_json_path) + NativeHelpers.run_nuget_discover_tool(job_path: job_file_path, + repo_root: repo_contents_path, + workspace_path: directory, + output_path: discovery_json_path, + credentials: credentials) + + Dependabot.logger.info("Discovery JSON content: #{File.read(discovery_json_path)}") end + load_discovery_for_directory(repo_contents_path: repo_contents_path, directory: directory) + end - # Loads NuGet dependency discovery for the given directory and returns a new instance of - # NativeDiscoveryJsonReader and caches the resultant object. - sig { params(repo_contents_path: String, directory: String).returns(NativeDiscoveryJsonReader) } - def self.load_discovery_for_directory(repo_contents_path:, directory:) - cache_directory_to_discovery_json_reader[directory] ||= begin - discovery_json_reader = discovery_json_reader(repo_contents_path: repo_contents_path, - workspace_path: directory) - cache_directory_to_discovery_json_reader[directory] = discovery_json_reader - dependency_file_cache_key = cache_key_from_dependency_file_paths(discovery_json_reader.dependency_file_paths) - cache_dependency_file_paths_to_discovery_json_reader[dependency_file_cache_key] = discovery_json_reader - discovery_file_path = discovery_file_path_from_workspace_path(directory) - cache_dependency_file_paths_to_discovery_json_path[dependency_file_cache_key] = discovery_file_path - - discovery_json_reader - end + # Loads NuGet dependency discovery for the given directory and returns a new instance of + # NativeDiscoveryJsonReader and caches the resultant object. + sig { params(repo_contents_path: String, directory: String).returns(NativeDiscoveryJsonReader) } + def self.load_discovery_for_directory(repo_contents_path:, directory:) + cache_directory_to_discovery_json_reader[directory] ||= begin + discovery_json_reader = discovery_json_reader(repo_contents_path: repo_contents_path, + workspace_path: directory) + cache_directory_to_discovery_json_reader[directory] = discovery_json_reader + dependency_file_cache_key = cache_key_from_dependency_file_paths(discovery_json_reader.dependency_file_paths) + cache_dependency_file_paths_to_discovery_json_reader[dependency_file_cache_key] = discovery_json_reader + discovery_file_path = discovery_file_path_from_workspace_path(directory) + cache_dependency_file_paths_to_discovery_json_path[dependency_file_cache_key] = discovery_file_path + + discovery_json_reader end + end - # Retrieves the cached NativeDiscoveryJsonReader object for the given dependency file paths. - sig { params(dependency_file_paths: T::Array[String]).returns(NativeDiscoveryJsonReader) } - def self.load_discovery_for_dependency_file_paths(dependency_file_paths) - dependency_file_cache_key = cache_key_from_dependency_file_paths(dependency_file_paths) - T.must(cache_dependency_file_paths_to_discovery_json_reader[dependency_file_cache_key]) - end + # Retrieves the cached NativeDiscoveryJsonReader object for the given dependency file paths. + sig { params(dependency_file_paths: T::Array[String]).returns(NativeDiscoveryJsonReader) } + def self.load_discovery_for_dependency_file_paths(dependency_file_paths) + dependency_file_cache_key = cache_key_from_dependency_file_paths(dependency_file_paths) + T.must(cache_dependency_file_paths_to_discovery_json_reader[dependency_file_cache_key]) + end - # Retrieves the cached location of the discovery JSON file for the given dependency file paths. - sig { params(dependency_file_paths: T::Array[String]).returns(String) } - def self.get_discovery_json_path_for_dependency_file_paths(dependency_file_paths) - dependency_file_cache_key = cache_key_from_dependency_file_paths(dependency_file_paths) - T.must(cache_dependency_file_paths_to_discovery_json_path[dependency_file_cache_key]) - end + # Retrieves the cached location of the discovery JSON file for the given dependency file paths. + sig { params(dependency_file_paths: T::Array[String]).returns(String) } + def self.get_discovery_json_path_for_dependency_file_paths(dependency_file_paths) + dependency_file_cache_key = cache_key_from_dependency_file_paths(dependency_file_paths) + T.must(cache_dependency_file_paths_to_discovery_json_path[dependency_file_cache_key]) + end - sig { params(repo_contents_path: String, dependency_file: Dependabot::DependencyFile).returns(String) } - def self.dependency_file_path(repo_contents_path:, dependency_file:) - dep_file_path = Pathname.new(File.join(dependency_file.directory, dependency_file.name)).cleanpath.to_path - dep_file_path.delete_prefix("#{repo_contents_path}/") - end + sig { params(repo_contents_path: String, dependency_file: Dependabot::DependencyFile).returns(String) } + def self.dependency_file_path(repo_contents_path:, dependency_file:) + dep_file_path = Pathname.new(File.join(dependency_file.directory, dependency_file.name)).cleanpath.to_path + dep_file_path.delete_prefix("#{repo_contents_path}/") + end + + sig { returns(String) } + def self.discovery_map_file_path + File.join(discovery_directory, "discovery_map.json") + end - sig { returns(String) } - def self.discovery_map_file_path - File.join(discovery_directory, "discovery_map.json") + sig { params(workspace_path: String).returns(String) } + def self.discovery_file_path_from_workspace_path(workspace_path) + # Given an update directory (also known as a workspace path), this function returns the path where the discovery + # JSON file is located. This function is called both by methods that need to write the discovery JSON file and + # by methods that need to read the discovery JSON file. This function is also called by multiple processes so + # we need a way to retain the data. This is accomplished by the following steps: + # 1. Check a well-known file for a mapping of workspace_path => discovery file path. If found, return it. + # 2. If the path is not found, generate a new path, save it to the well-known file, and return the value. + discovery_map_contents = File.exist?(discovery_map_file_path) ? File.read(discovery_map_file_path) : "{}" + discovery_map = T.let(JSON.parse(discovery_map_contents), T::Hash[String, String]) + + discovery_json_path = discovery_map[workspace_path] + if discovery_json_path + Dependabot.logger.info("Discovery JSON path for workspace path [#{workspace_path}] found in file " \ + "[#{discovery_map_file_path}] at location [#{discovery_json_path}]") + return discovery_json_path end - sig { params(workspace_path: String).returns(String) } - def self.discovery_file_path_from_workspace_path(workspace_path) - # Given an update directory (also known as a workspace path), this function returns the path where the discovery - # JSON file is located. This function is called both by methods that need to write the discovery JSON file and - # by methods that need to read the discovery JSON file. This function is also called by multiple processes so - # we need a way to retain the data. This is accomplished by the following steps: - # 1. Check a well-known file for a mapping of workspace_path => discovery file path. If found, return it. - # 2. If the path is not found, generate a new path, save it to the well-known file, and return the value. - discovery_map_contents = File.exist?(discovery_map_file_path) ? File.read(discovery_map_file_path) : "{}" - discovery_map = T.let(JSON.parse(discovery_map_contents), T::Hash[String, String]) - - discovery_json_path = discovery_map[workspace_path] - if discovery_json_path - Dependabot.logger.info("Discovery JSON path for workspace path [#{workspace_path}] found in file " \ - "[#{discovery_map_file_path}] at location [#{discovery_json_path}]") - return discovery_json_path - end + # no discovery JSON path found; generate a new one, but first find a suitable location + discovery_json_counter = 1 + new_discovery_json_path = "" + loop do + new_discovery_json_path = File.join(discovery_directory, "discovery.#{discovery_json_counter}.json") + break unless File.exist?(new_discovery_json_path) + + discovery_json_counter += 1 + end - # no discovery JSON path found; generate a new one, but first find a suitable location - discovery_json_counter = 1 - new_discovery_json_path = "" - loop do - new_discovery_json_path = File.join(discovery_directory, "discovery.#{discovery_json_counter}.json") - break unless File.exist?(new_discovery_json_path) + discovery_map[workspace_path] = new_discovery_json_path - discovery_json_counter += 1 - end + File.write(discovery_map_file_path, discovery_map.to_json) + Dependabot.logger.info("Discovery JSON path for workspace path [#{workspace_path}] created for file " \ + "[#{discovery_map_file_path}] at location [#{new_discovery_json_path}]") + new_discovery_json_path + end - discovery_map[workspace_path] = new_discovery_json_path + sig { params(dependency_file_paths: T::Array[String]).returns(String) } + def self.cache_key_from_dependency_file_paths(dependency_file_paths) + dependency_file_paths.sort.join(",") + end - File.write(discovery_map_file_path, discovery_map.to_json) - Dependabot.logger.info("Discovery JSON path for workspace path [#{workspace_path}] created for file " \ - "[#{discovery_map_file_path}] at location [#{new_discovery_json_path}]") - new_discovery_json_path - end + sig { returns(String) } + def self.discovery_directory + t = File.join(Dir.home, ".dependabot") + FileUtils.mkdir_p(t) + t + end - sig { params(dependency_file_paths: T::Array[String]).returns(String) } - def self.cache_key_from_dependency_file_paths(dependency_file_paths) - dependency_file_paths.sort.join(",") - end + sig { params(repo_contents_path: String, workspace_path: String).returns(NativeDiscoveryJsonReader) } + def self.discovery_json_reader(repo_contents_path:, workspace_path:) + discovery_file_path = discovery_file_path_from_workspace_path(workspace_path) + discovery_json = DependencyFile.new( + name: Pathname.new(discovery_file_path).cleanpath.to_path, + directory: discovery_directory, + type: "file", + content: File.read(discovery_file_path) + ) + NativeDiscoveryJsonReader.new(repo_contents_path: repo_contents_path, discovery_json: discovery_json) + end - sig { returns(String) } - def self.discovery_directory - t = File.join(Dir.home, ".dependabot") - FileUtils.mkdir_p(t) - t - end + sig { returns(T.nilable(NativeWorkspaceDiscovery)) } + attr_reader :workspace_discovery - sig { params(repo_contents_path: String, workspace_path: String).returns(NativeDiscoveryJsonReader) } - def self.discovery_json_reader(repo_contents_path:, workspace_path:) - discovery_file_path = discovery_file_path_from_workspace_path(workspace_path) - discovery_json = DependencyFile.new( - name: Pathname.new(discovery_file_path).cleanpath.to_path, - directory: discovery_directory, - type: "file", - content: File.read(discovery_file_path) - ) - NativeDiscoveryJsonReader.new(repo_contents_path: repo_contents_path, discovery_json: discovery_json) - end + sig { returns(Dependabot::FileParsers::Base::DependencySet) } + attr_reader :dependency_set - sig { returns(T.nilable(NativeWorkspaceDiscovery)) } - attr_reader :workspace_discovery + sig { returns(T::Array[String]) } + attr_reader :dependency_file_paths - sig { returns(Dependabot::FileParsers::Base::DependencySet) } - attr_reader :dependency_set + sig { params(repo_contents_path: String, discovery_json: DependencyFile).void } + def initialize(repo_contents_path:, discovery_json:) + @repo_contents_path = repo_contents_path + @discovery_json = discovery_json + @workspace_discovery = T.let(read_workspace_discovery, T.nilable(Dependabot::Nuget::NativeWorkspaceDiscovery)) + @dependency_set = T.let(read_dependency_set, Dependabot::FileParsers::Base::DependencySet) + @dependency_file_paths = T.let(read_dependency_file_paths, T::Array[String]) + end - sig { returns(T::Array[String]) } - attr_reader :dependency_file_paths + private - sig { params(repo_contents_path: String, discovery_json: DependencyFile).void } - def initialize(repo_contents_path:, discovery_json:) - @repo_contents_path = repo_contents_path - @discovery_json = discovery_json - @workspace_discovery = T.let(read_workspace_discovery, T.nilable(Dependabot::Nuget::NativeWorkspaceDiscovery)) - @dependency_set = T.let(read_dependency_set, Dependabot::FileParsers::Base::DependencySet) - @dependency_file_paths = T.let(read_dependency_file_paths, T::Array[String]) - end + sig { returns(String) } + attr_reader :repo_contents_path - private + sig { returns(DependencyFile) } + attr_reader :discovery_json - sig { returns(String) } - attr_reader :repo_contents_path + sig { returns(T.nilable(NativeWorkspaceDiscovery)) } + def read_workspace_discovery + return nil unless discovery_json.content - sig { returns(DependencyFile) } - attr_reader :discovery_json + parsed_json = T.let(JSON.parse(T.must(discovery_json.content)), T::Hash[String, T.untyped]) + NativeWorkspaceDiscovery.from_json(parsed_json) + rescue JSON::ParserError + raise Dependabot::DependencyFileNotParseable, discovery_json.path + end - sig { returns(T.nilable(NativeWorkspaceDiscovery)) } - def read_workspace_discovery - return nil unless discovery_json.content + sig { returns(Dependabot::FileParsers::Base::DependencySet) } + def read_dependency_set + dependency_set = Dependabot::FileParsers::Base::DependencySet.new + return dependency_set unless workspace_discovery - parsed_json = T.let(JSON.parse(T.must(discovery_json.content)), T::Hash[String, T.untyped]) - NativeWorkspaceDiscovery.from_json(parsed_json) - rescue JSON::ParserError - raise Dependabot::DependencyFileNotParseable, discovery_json.path + workspace_result = T.must(workspace_discovery) + workspace_result.projects.each do |project| + dependency_set += project.dependency_set end + if workspace_result.dotnet_tools_json + dependency_set += T.must(workspace_result.dotnet_tools_json).dependency_set + end + dependency_set += T.must(workspace_result.global_json).dependency_set if workspace_result.global_json - sig { returns(Dependabot::FileParsers::Base::DependencySet) } - def read_dependency_set - dependency_set = Dependabot::FileParsers::Base::DependencySet.new - return dependency_set unless workspace_discovery - - workspace_result = T.must(workspace_discovery) - workspace_result.projects.each do |project| - dependency_set += project.dependency_set - end - if workspace_result.dotnet_tools_json - dependency_set += T.must(workspace_result.dotnet_tools_json).dependency_set - end - dependency_set += T.must(workspace_result.global_json).dependency_set if workspace_result.global_json + dependency_set + end - dependency_set + sig { returns(T::Array[String]) } + def read_dependency_file_paths + dependency_file_paths = T.let([], T::Array[T.nilable(String)]) + dependency_file_paths << dependency_file_path_from_repo_path("global.json") if workspace_discovery&.global_json + if workspace_discovery&.dotnet_tools_json + dependency_file_paths << dependency_file_path_from_repo_path(".config/dotnet-tools.json") end - sig { returns(T::Array[String]) } - def read_dependency_file_paths - dependency_file_paths = T.let([], T::Array[T.nilable(String)]) - if workspace_discovery&.global_json - dependency_file_paths << dependency_file_path_from_repo_path("global.json") - end - if workspace_discovery&.dotnet_tools_json - dependency_file_paths << dependency_file_path_from_repo_path(".config/dotnet-tools.json") + projects = workspace_discovery&.projects || [] + projects.each do |project| + dependency_file_paths << dependency_file_path_from_repo_path(project.file_path) + dependency_file_paths += project.imported_files.map do |f| + dependency_file_path_from_project_path(project.file_path, f) end - - projects = workspace_discovery&.projects || [] - projects.each do |project| - dependency_file_paths << dependency_file_path_from_repo_path(project.file_path) - dependency_file_paths += project.imported_files.map do |f| - dependency_file_path_from_project_path(project.file_path, f) - end - dependency_file_paths += project.additional_files.map do |f| - dependency_file_path_from_project_path(project.file_path, f) - end + dependency_file_paths += project.additional_files.map do |f| + dependency_file_path_from_project_path(project.file_path, f) end - - deduped_dependency_file_paths = T.let(Set.new(dependency_file_paths.compact), T::Set[String]) - result = deduped_dependency_file_paths.sort - result end - sig { params(path_parts: String).returns(T.nilable(String)) } - def dependency_file_path_from_repo_path(*path_parts) - path_parts = path_parts.map { |p| p.delete_prefix("/").delete_suffix("/") } - normalized_repo_path = Pathname.new(path_parts.join("/")).cleanpath.to_path.delete_prefix("/") - full_path = Pathname.new(File.join(repo_contents_path, normalized_repo_path)).cleanpath.to_path - return unless File.exist?(full_path) + deduped_dependency_file_paths = T.let(Set.new(dependency_file_paths.compact), T::Set[String]) + result = deduped_dependency_file_paths.sort + result + end - normalized_repo_path = "/#{normalized_repo_path}" unless normalized_repo_path.start_with?("/") - normalized_repo_path - end + sig { params(path_parts: String).returns(T.nilable(String)) } + def dependency_file_path_from_repo_path(*path_parts) + path_parts = path_parts.map { |p| p.delete_prefix("/").delete_suffix("/") } + normalized_repo_path = Pathname.new(path_parts.join("/")).cleanpath.to_path.delete_prefix("/") + full_path = Pathname.new(File.join(repo_contents_path, normalized_repo_path)).cleanpath.to_path + return unless File.exist?(full_path) - sig { params(project_path: String, relative_file_path: String).returns(T.nilable(String)) } - def dependency_file_path_from_project_path(project_path, relative_file_path) - project_directory = File.dirname(project_path) - dependency_file_path_from_repo_path(project_directory, relative_file_path) - end + normalized_repo_path = "/#{normalized_repo_path}" unless normalized_repo_path.start_with?("/") + normalized_repo_path + end + + sig { params(project_path: String, relative_file_path: String).returns(T.nilable(String)) } + def dependency_file_path_from_project_path(project_path, relative_file_path) + project_directory = File.dirname(project_path) + dependency_file_path_from_repo_path(project_directory, relative_file_path) end end end diff --git a/nuget/lib/dependabot/nuget/native_update_checker/native_update_checker.rb b/nuget/lib/dependabot/nuget/native_update_checker/native_update_checker.rb index 32ab30da5c..10fe1034c5 100644 --- a/nuget/lib/dependabot/nuget/native_update_checker/native_update_checker.rb +++ b/nuget/lib/dependabot/nuget/native_update_checker/native_update_checker.rb @@ -97,7 +97,7 @@ def dependency_file_path sig { returns(T::Array[String]) } def dependency_file_paths dependency_files.map do |file| - Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.dependency_file_path( + NativeDiscoveryJsonReader.dependency_file_path( repo_contents_path: T.must(repo_contents_path), dependency_file: file ) @@ -106,7 +106,7 @@ def dependency_file_paths sig { returns(AnalysisJsonReader) } def request_analysis - discovery_file_path = Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.get_discovery_json_path_for_dependency_file_paths( + discovery_file_path = NativeDiscoveryJsonReader.get_discovery_json_path_for_dependency_file_paths( dependency_file_paths ) analysis_folder_path = AnalysisJsonReader.temp_directory @@ -154,7 +154,7 @@ def write_dependency_info sig { returns(Dependabot::FileParsers::Base::DependencySet) } def discovered_dependencies - Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.load_discovery_for_dependency_file_paths(dependency_file_paths).dependency_set + NativeDiscoveryJsonReader.load_discovery_for_dependency_file_paths(dependency_file_paths).dependency_set end sig { override.returns(T::Boolean) } From 56ba52fc1383a7e2b41915c1e0619b288017e610 Mon Sep 17 00:00:00 2001 From: sachin-sandhu Date: Wed, 11 Dec 2024 10:34:26 -0500 Subject: [PATCH 10/11] new commit to test sorbet fixes --- nuget/lib/dependabot/nuget/file_fetcher.rb | 2 +- nuget/lib/dependabot/nuget/file_parser.rb | 4 +- nuget/lib/dependabot/nuget/file_updater.rb | 12 +- .../native_discovery_json_reader.rb | 420 +++++++++--------- .../native_update_checker.rb | 6 +- 5 files changed, 224 insertions(+), 220 deletions(-) diff --git a/nuget/lib/dependabot/nuget/file_fetcher.rb b/nuget/lib/dependabot/nuget/file_fetcher.rb index 5dacfd983e..3ad640b7b2 100644 --- a/nuget/lib/dependabot/nuget/file_fetcher.rb +++ b/nuget/lib/dependabot/nuget/file_fetcher.rb @@ -26,7 +26,7 @@ def self.required_files_message sig { override.returns(T::Array[DependencyFile]) } def fetch_files - discovery_json_reader = NativeDiscoveryJsonReader.run_discovery_in_directory( + discovery_json_reader = Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.run_discovery_in_directory( repo_contents_path: T.must(repo_contents_path), directory: directory, credentials: credentials diff --git a/nuget/lib/dependabot/nuget/file_parser.rb b/nuget/lib/dependabot/nuget/file_parser.rb index c68b06376d..e3adf0c561 100644 --- a/nuget/lib/dependabot/nuget/file_parser.rb +++ b/nuget/lib/dependabot/nuget/file_parser.rb @@ -41,7 +41,7 @@ def ecosystem def content_json @content_json ||= T.let(begin directory = source&.directory || "/" - discovery_json_reader = Dependabot::Nuget::NativeDiscoveryJsonReader.run_discovery_in_directory( + discovery_json_reader = Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.run_discovery_in_directory( repo_contents_path: T.must(repo_contents_path), directory: directory, credentials: credentials @@ -54,7 +54,7 @@ def content_json def dependencies @dependencies ||= T.let(begin directory = source&.directory || "/" - discovery_json_reader = NativeDiscoveryJsonReader.run_discovery_in_directory( + discovery_json_reader = Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.run_discovery_in_directory( repo_contents_path: T.must(repo_contents_path), directory: directory, credentials: credentials diff --git a/nuget/lib/dependabot/nuget/file_updater.rb b/nuget/lib/dependabot/nuget/file_updater.rb index 39393b9ae5..43be9c529b 100644 --- a/nuget/lib/dependabot/nuget/file_updater.rb +++ b/nuget/lib/dependabot/nuget/file_updater.rb @@ -57,7 +57,7 @@ def updated_dependency_files try_update_projects(dependency) || try_update_json(dependency) end updated_files = dependency_files.filter_map do |f| - dependency_file_path = NativeDiscoveryJsonReader.dependency_file_path( + dependency_file_path = Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.dependency_file_path( repo_contents_path: T.must(repo_contents_path), dependency_file: f ) @@ -97,7 +97,7 @@ def try_update_projects(dependency) # run update for each project file project_files.each do |project_file| project_dependencies = project_dependencies(project_file) - dependency_file_path = NativeDiscoveryJsonReader.dependency_file_path( + dependency_file_path = Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.dependency_file_path( repo_contents_path: T.must(repo_contents_path), dependency_file: project_file ) @@ -128,7 +128,7 @@ def try_update_json(dependency) # We just need to feed the updater a project file, grab the first project_file = T.must(project_files.first) - dependency_file_path = NativeDiscoveryJsonReader.dependency_file_path( + dependency_file_path = Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.dependency_file_path( repo_contents_path: T.must(repo_contents_path), dependency_file: project_file ) @@ -171,10 +171,10 @@ def testonly_update_tooling_calls sig { returns(T.nilable(NativeWorkspaceDiscovery)) } def workspace dependency_file_paths = dependency_files.map do |f| - NativeDiscoveryJsonReader.dependency_file_path(repo_contents_path: T.must(repo_contents_path), - dependency_file: f) + Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.dependency_file_path(repo_contents_path: T.must(repo_contents_path), + dependency_file: f) end - NativeDiscoveryJsonReader.load_discovery_for_dependency_file_paths(dependency_file_paths).workspace_discovery + Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.load_discovery_for_dependency_file_paths(dependency_file_paths).workspace_discovery end sig { params(project_file: Dependabot::DependencyFile).returns(T::Array[String]) } diff --git a/nuget/lib/dependabot/nuget/native_discovery/native_discovery_json_reader.rb b/nuget/lib/dependabot/nuget/native_discovery/native_discovery_json_reader.rb index 7bcaadb865..cad93b4d3a 100644 --- a/nuget/lib/dependabot/nuget/native_discovery/native_discovery_json_reader.rb +++ b/nuget/lib/dependabot/nuget/native_discovery/native_discovery_json_reader.rb @@ -10,256 +10,260 @@ module Dependabot module Nuget - class NativeDiscoveryJsonReader - extend T::Sig + module NativeDiscovery + class NativeDiscoveryJsonReader + extend T::Sig - sig { returns(T::Hash[String, NativeDiscoveryJsonReader]) } - def self.cache_directory_to_discovery_json_reader - CacheManager.cache("cache_directory_to_discovery_json_reader") - end - - sig { returns(T::Hash[String, NativeDiscoveryJsonReader]) } - def self.cache_dependency_file_paths_to_discovery_json_reader - CacheManager.cache("cache_dependency_file_paths_to_discovery_json_reader") - end - - sig { returns(T::Hash[String, String]) } - def self.cache_dependency_file_paths_to_discovery_json_path - CacheManager.cache("cache_dependency_file_paths_to_discovery_json_path") - end - - sig { void } - def self.testonly_clear_caches - cache_directory_to_discovery_json_reader.clear - cache_dependency_file_paths_to_discovery_json_reader.clear - cache_dependency_file_paths_to_discovery_json_path.clear - end + sig { returns(T::Hash[String, NativeDiscoveryJsonReader]) } + def self.cache_directory_to_discovery_json_reader + CacheManager.cache("cache_directory_to_discovery_json_reader") + end - sig { void } - def self.testonly_clear_discovery_files - # this will get recreated when necessary - FileUtils.rm_rf(discovery_directory) - end + sig { returns(T::Hash[String, NativeDiscoveryJsonReader]) } + def self.cache_dependency_file_paths_to_discovery_json_reader + CacheManager.cache("cache_dependency_file_paths_to_discovery_json_reader") + end - # Runs NuGet dependency discovery in the given directory and returns a new instance of NativeDiscoveryJsonReader. - # The location of the resultant JSON file is saved. - sig do - params( - repo_contents_path: String, - directory: String, - credentials: T::Array[Dependabot::Credential] - ).returns(NativeDiscoveryJsonReader) - end - def self.run_discovery_in_directory(repo_contents_path:, directory:, credentials:) - # run discovery - job_file_path = ENV.fetch("DEPENDABOT_JOB_PATH") - discovery_json_path = discovery_file_path_from_workspace_path(directory) - unless File.exist?(discovery_json_path) - NativeHelpers.run_nuget_discover_tool(job_path: job_file_path, - repo_root: repo_contents_path, - workspace_path: directory, - output_path: discovery_json_path, - credentials: credentials) - - Dependabot.logger.info("Discovery JSON content: #{File.read(discovery_json_path)}") + sig { returns(T::Hash[String, String]) } + def self.cache_dependency_file_paths_to_discovery_json_path + CacheManager.cache("cache_dependency_file_paths_to_discovery_json_path") end - load_discovery_for_directory(repo_contents_path: repo_contents_path, directory: directory) - end - # Loads NuGet dependency discovery for the given directory and returns a new instance of - # NativeDiscoveryJsonReader and caches the resultant object. - sig { params(repo_contents_path: String, directory: String).returns(NativeDiscoveryJsonReader) } - def self.load_discovery_for_directory(repo_contents_path:, directory:) - cache_directory_to_discovery_json_reader[directory] ||= begin - discovery_json_reader = discovery_json_reader(repo_contents_path: repo_contents_path, - workspace_path: directory) - cache_directory_to_discovery_json_reader[directory] = discovery_json_reader - dependency_file_cache_key = cache_key_from_dependency_file_paths(discovery_json_reader.dependency_file_paths) - cache_dependency_file_paths_to_discovery_json_reader[dependency_file_cache_key] = discovery_json_reader - discovery_file_path = discovery_file_path_from_workspace_path(directory) - cache_dependency_file_paths_to_discovery_json_path[dependency_file_cache_key] = discovery_file_path - - discovery_json_reader + sig { void } + def self.testonly_clear_caches + cache_directory_to_discovery_json_reader.clear + cache_dependency_file_paths_to_discovery_json_reader.clear + cache_dependency_file_paths_to_discovery_json_path.clear end - end - # Retrieves the cached NativeDiscoveryJsonReader object for the given dependency file paths. - sig { params(dependency_file_paths: T::Array[String]).returns(NativeDiscoveryJsonReader) } - def self.load_discovery_for_dependency_file_paths(dependency_file_paths) - dependency_file_cache_key = cache_key_from_dependency_file_paths(dependency_file_paths) - T.must(cache_dependency_file_paths_to_discovery_json_reader[dependency_file_cache_key]) - end + sig { void } + def self.testonly_clear_discovery_files + # this will get recreated when necessary + FileUtils.rm_rf(discovery_directory) + end - # Retrieves the cached location of the discovery JSON file for the given dependency file paths. - sig { params(dependency_file_paths: T::Array[String]).returns(String) } - def self.get_discovery_json_path_for_dependency_file_paths(dependency_file_paths) - dependency_file_cache_key = cache_key_from_dependency_file_paths(dependency_file_paths) - T.must(cache_dependency_file_paths_to_discovery_json_path[dependency_file_cache_key]) - end + # Runs NuGet dependency discovery in the given directory and returns a new instance of NativeDiscoveryJsonReader. + # The location of the resultant JSON file is saved. + sig do + params( + repo_contents_path: String, + directory: String, + credentials: T::Array[Dependabot::Credential] + ).returns(NativeDiscoveryJsonReader) + end + def self.run_discovery_in_directory(repo_contents_path:, directory:, credentials:) + # run discovery + job_file_path = ENV.fetch("DEPENDABOT_JOB_PATH") + discovery_json_path = discovery_file_path_from_workspace_path(directory) + unless File.exist?(discovery_json_path) + NativeHelpers.run_nuget_discover_tool(job_path: job_file_path, + repo_root: repo_contents_path, + workspace_path: directory, + output_path: discovery_json_path, + credentials: credentials) + + Dependabot.logger.info("Discovery JSON content: #{File.read(discovery_json_path)}") + end + load_discovery_for_directory(repo_contents_path: repo_contents_path, directory: directory) + end - sig { params(repo_contents_path: String, dependency_file: Dependabot::DependencyFile).returns(String) } - def self.dependency_file_path(repo_contents_path:, dependency_file:) - dep_file_path = Pathname.new(File.join(dependency_file.directory, dependency_file.name)).cleanpath.to_path - dep_file_path.delete_prefix("#{repo_contents_path}/") - end + # Loads NuGet dependency discovery for the given directory and returns a new instance of + # NativeDiscoveryJsonReader and caches the resultant object. + sig { params(repo_contents_path: String, directory: String).returns(NativeDiscoveryJsonReader) } + def self.load_discovery_for_directory(repo_contents_path:, directory:) + cache_directory_to_discovery_json_reader[directory] ||= begin + discovery_json_reader = discovery_json_reader(repo_contents_path: repo_contents_path, + workspace_path: directory) + cache_directory_to_discovery_json_reader[directory] = discovery_json_reader + dependency_file_cache_key = cache_key_from_dependency_file_paths(discovery_json_reader.dependency_file_paths) + cache_dependency_file_paths_to_discovery_json_reader[dependency_file_cache_key] = discovery_json_reader + discovery_file_path = discovery_file_path_from_workspace_path(directory) + cache_dependency_file_paths_to_discovery_json_path[dependency_file_cache_key] = discovery_file_path + + discovery_json_reader + end + end - sig { returns(String) } - def self.discovery_map_file_path - File.join(discovery_directory, "discovery_map.json") - end + # Retrieves the cached NativeDiscoveryJsonReader object for the given dependency file paths. + sig { params(dependency_file_paths: T::Array[String]).returns(NativeDiscoveryJsonReader) } + def self.load_discovery_for_dependency_file_paths(dependency_file_paths) + dependency_file_cache_key = cache_key_from_dependency_file_paths(dependency_file_paths) + T.must(cache_dependency_file_paths_to_discovery_json_reader[dependency_file_cache_key]) + end - sig { params(workspace_path: String).returns(String) } - def self.discovery_file_path_from_workspace_path(workspace_path) - # Given an update directory (also known as a workspace path), this function returns the path where the discovery - # JSON file is located. This function is called both by methods that need to write the discovery JSON file and - # by methods that need to read the discovery JSON file. This function is also called by multiple processes so - # we need a way to retain the data. This is accomplished by the following steps: - # 1. Check a well-known file for a mapping of workspace_path => discovery file path. If found, return it. - # 2. If the path is not found, generate a new path, save it to the well-known file, and return the value. - discovery_map_contents = File.exist?(discovery_map_file_path) ? File.read(discovery_map_file_path) : "{}" - discovery_map = T.let(JSON.parse(discovery_map_contents), T::Hash[String, String]) - - discovery_json_path = discovery_map[workspace_path] - if discovery_json_path - Dependabot.logger.info("Discovery JSON path for workspace path [#{workspace_path}] found in file " \ - "[#{discovery_map_file_path}] at location [#{discovery_json_path}]") - return discovery_json_path + # Retrieves the cached location of the discovery JSON file for the given dependency file paths. + sig { params(dependency_file_paths: T::Array[String]).returns(String) } + def self.get_discovery_json_path_for_dependency_file_paths(dependency_file_paths) + dependency_file_cache_key = cache_key_from_dependency_file_paths(dependency_file_paths) + T.must(cache_dependency_file_paths_to_discovery_json_path[dependency_file_cache_key]) end - # no discovery JSON path found; generate a new one, but first find a suitable location - discovery_json_counter = 1 - new_discovery_json_path = "" - loop do - new_discovery_json_path = File.join(discovery_directory, "discovery.#{discovery_json_counter}.json") - break unless File.exist?(new_discovery_json_path) + sig { params(repo_contents_path: String, dependency_file: Dependabot::DependencyFile).returns(String) } + def self.dependency_file_path(repo_contents_path:, dependency_file:) + dep_file_path = Pathname.new(File.join(dependency_file.directory, dependency_file.name)).cleanpath.to_path + dep_file_path.delete_prefix("#{repo_contents_path}/") + end - discovery_json_counter += 1 + sig { returns(String) } + def self.discovery_map_file_path + File.join(discovery_directory, "discovery_map.json") end - discovery_map[workspace_path] = new_discovery_json_path + sig { params(workspace_path: String).returns(String) } + def self.discovery_file_path_from_workspace_path(workspace_path) + # Given an update directory (also known as a workspace path), this function returns the path where the discovery + # JSON file is located. This function is called both by methods that need to write the discovery JSON file and + # by methods that need to read the discovery JSON file. This function is also called by multiple processes so + # we need a way to retain the data. This is accomplished by the following steps: + # 1. Check a well-known file for a mapping of workspace_path => discovery file path. If found, return it. + # 2. If the path is not found, generate a new path, save it to the well-known file, and return the value. + discovery_map_contents = File.exist?(discovery_map_file_path) ? File.read(discovery_map_file_path) : "{}" + discovery_map = T.let(JSON.parse(discovery_map_contents), T::Hash[String, String]) + + discovery_json_path = discovery_map[workspace_path] + if discovery_json_path + Dependabot.logger.info("Discovery JSON path for workspace path [#{workspace_path}] found in file " \ + "[#{discovery_map_file_path}] at location [#{discovery_json_path}]") + return discovery_json_path + end + + # no discovery JSON path found; generate a new one, but first find a suitable location + discovery_json_counter = 1 + new_discovery_json_path = "" + loop do + new_discovery_json_path = File.join(discovery_directory, "discovery.#{discovery_json_counter}.json") + break unless File.exist?(new_discovery_json_path) - File.write(discovery_map_file_path, discovery_map.to_json) - Dependabot.logger.info("Discovery JSON path for workspace path [#{workspace_path}] created for file " \ - "[#{discovery_map_file_path}] at location [#{new_discovery_json_path}]") - new_discovery_json_path - end + discovery_json_counter += 1 + end - sig { params(dependency_file_paths: T::Array[String]).returns(String) } - def self.cache_key_from_dependency_file_paths(dependency_file_paths) - dependency_file_paths.sort.join(",") - end + discovery_map[workspace_path] = new_discovery_json_path - sig { returns(String) } - def self.discovery_directory - t = File.join(Dir.home, ".dependabot") - FileUtils.mkdir_p(t) - t - end + File.write(discovery_map_file_path, discovery_map.to_json) + Dependabot.logger.info("Discovery JSON path for workspace path [#{workspace_path}] created for file " \ + "[#{discovery_map_file_path}] at location [#{new_discovery_json_path}]") + new_discovery_json_path + end - sig { params(repo_contents_path: String, workspace_path: String).returns(NativeDiscoveryJsonReader) } - def self.discovery_json_reader(repo_contents_path:, workspace_path:) - discovery_file_path = discovery_file_path_from_workspace_path(workspace_path) - discovery_json = DependencyFile.new( - name: Pathname.new(discovery_file_path).cleanpath.to_path, - directory: discovery_directory, - type: "file", - content: File.read(discovery_file_path) - ) - NativeDiscoveryJsonReader.new(repo_contents_path: repo_contents_path, discovery_json: discovery_json) - end + sig { params(dependency_file_paths: T::Array[String]).returns(String) } + def self.cache_key_from_dependency_file_paths(dependency_file_paths) + dependency_file_paths.sort.join(",") + end - sig { returns(T.nilable(NativeWorkspaceDiscovery)) } - attr_reader :workspace_discovery + sig { returns(String) } + def self.discovery_directory + t = File.join(Dir.home, ".dependabot") + FileUtils.mkdir_p(t) + t + end - sig { returns(Dependabot::FileParsers::Base::DependencySet) } - attr_reader :dependency_set + sig { params(repo_contents_path: String, workspace_path: String).returns(NativeDiscoveryJsonReader) } + def self.discovery_json_reader(repo_contents_path:, workspace_path:) + discovery_file_path = discovery_file_path_from_workspace_path(workspace_path) + discovery_json = DependencyFile.new( + name: Pathname.new(discovery_file_path).cleanpath.to_path, + directory: discovery_directory, + type: "file", + content: File.read(discovery_file_path) + ) + NativeDiscoveryJsonReader.new(repo_contents_path: repo_contents_path, discovery_json: discovery_json) + end - sig { returns(T::Array[String]) } - attr_reader :dependency_file_paths + sig { returns(T.nilable(NativeWorkspaceDiscovery)) } + attr_reader :workspace_discovery - sig { params(repo_contents_path: String, discovery_json: DependencyFile).void } - def initialize(repo_contents_path:, discovery_json:) - @repo_contents_path = repo_contents_path - @discovery_json = discovery_json - @workspace_discovery = T.let(read_workspace_discovery, T.nilable(Dependabot::Nuget::NativeWorkspaceDiscovery)) - @dependency_set = T.let(read_dependency_set, Dependabot::FileParsers::Base::DependencySet) - @dependency_file_paths = T.let(read_dependency_file_paths, T::Array[String]) - end + sig { returns(Dependabot::FileParsers::Base::DependencySet) } + attr_reader :dependency_set - private + sig { returns(T::Array[String]) } + attr_reader :dependency_file_paths - sig { returns(String) } - attr_reader :repo_contents_path + sig { params(repo_contents_path: String, discovery_json: DependencyFile).void } + def initialize(repo_contents_path:, discovery_json:) + @repo_contents_path = repo_contents_path + @discovery_json = discovery_json + @workspace_discovery = T.let(read_workspace_discovery, T.nilable(Dependabot::Nuget::NativeWorkspaceDiscovery)) + @dependency_set = T.let(read_dependency_set, Dependabot::FileParsers::Base::DependencySet) + @dependency_file_paths = T.let(read_dependency_file_paths, T::Array[String]) + end - sig { returns(DependencyFile) } - attr_reader :discovery_json + private - sig { returns(T.nilable(NativeWorkspaceDiscovery)) } - def read_workspace_discovery - return nil unless discovery_json.content + sig { returns(String) } + attr_reader :repo_contents_path - parsed_json = T.let(JSON.parse(T.must(discovery_json.content)), T::Hash[String, T.untyped]) - NativeWorkspaceDiscovery.from_json(parsed_json) - rescue JSON::ParserError - raise Dependabot::DependencyFileNotParseable, discovery_json.path - end + sig { returns(DependencyFile) } + attr_reader :discovery_json - sig { returns(Dependabot::FileParsers::Base::DependencySet) } - def read_dependency_set - dependency_set = Dependabot::FileParsers::Base::DependencySet.new - return dependency_set unless workspace_discovery + sig { returns(T.nilable(NativeWorkspaceDiscovery)) } + def read_workspace_discovery + return nil unless discovery_json.content - workspace_result = T.must(workspace_discovery) - workspace_result.projects.each do |project| - dependency_set += project.dependency_set + parsed_json = T.let(JSON.parse(T.must(discovery_json.content)), T::Hash[String, T.untyped]) + NativeWorkspaceDiscovery.from_json(parsed_json) + rescue JSON::ParserError + raise Dependabot::DependencyFileNotParseable, discovery_json.path end - if workspace_result.dotnet_tools_json - dependency_set += T.must(workspace_result.dotnet_tools_json).dependency_set - end - dependency_set += T.must(workspace_result.global_json).dependency_set if workspace_result.global_json - dependency_set - end + sig { returns(Dependabot::FileParsers::Base::DependencySet) } + def read_dependency_set + dependency_set = Dependabot::FileParsers::Base::DependencySet.new + return dependency_set unless workspace_discovery - sig { returns(T::Array[String]) } - def read_dependency_file_paths - dependency_file_paths = T.let([], T::Array[T.nilable(String)]) - dependency_file_paths << dependency_file_path_from_repo_path("global.json") if workspace_discovery&.global_json - if workspace_discovery&.dotnet_tools_json - dependency_file_paths << dependency_file_path_from_repo_path(".config/dotnet-tools.json") + workspace_result = T.must(workspace_discovery) + workspace_result.projects.each do |project| + dependency_set += project.dependency_set + end + if workspace_result.dotnet_tools_json + dependency_set += T.must(workspace_result.dotnet_tools_json).dependency_set + end + dependency_set += T.must(workspace_result.global_json).dependency_set if workspace_result.global_json + + dependency_set end - projects = workspace_discovery&.projects || [] - projects.each do |project| - dependency_file_paths << dependency_file_path_from_repo_path(project.file_path) - dependency_file_paths += project.imported_files.map do |f| - dependency_file_path_from_project_path(project.file_path, f) + sig { returns(T::Array[String]) } + def read_dependency_file_paths + dependency_file_paths = T.let([], T::Array[T.nilable(String)]) + if workspace_discovery&.global_json + dependency_file_paths << dependency_file_path_from_repo_path("global.json") end - dependency_file_paths += project.additional_files.map do |f| - dependency_file_path_from_project_path(project.file_path, f) + if workspace_discovery&.dotnet_tools_json + dependency_file_paths << dependency_file_path_from_repo_path(".config/dotnet-tools.json") end - end - deduped_dependency_file_paths = T.let(Set.new(dependency_file_paths.compact), T::Set[String]) - result = deduped_dependency_file_paths.sort - result - end + projects = workspace_discovery&.projects || [] + projects.each do |project| + dependency_file_paths << dependency_file_path_from_repo_path(project.file_path) + dependency_file_paths += project.imported_files.map do |f| + dependency_file_path_from_project_path(project.file_path, f) + end + dependency_file_paths += project.additional_files.map do |f| + dependency_file_path_from_project_path(project.file_path, f) + end + end - sig { params(path_parts: String).returns(T.nilable(String)) } - def dependency_file_path_from_repo_path(*path_parts) - path_parts = path_parts.map { |p| p.delete_prefix("/").delete_suffix("/") } - normalized_repo_path = Pathname.new(path_parts.join("/")).cleanpath.to_path.delete_prefix("/") - full_path = Pathname.new(File.join(repo_contents_path, normalized_repo_path)).cleanpath.to_path - return unless File.exist?(full_path) + deduped_dependency_file_paths = T.let(Set.new(dependency_file_paths.compact), T::Set[String]) + result = deduped_dependency_file_paths.sort + result + end - normalized_repo_path = "/#{normalized_repo_path}" unless normalized_repo_path.start_with?("/") - normalized_repo_path - end + sig { params(path_parts: String).returns(T.nilable(String)) } + def dependency_file_path_from_repo_path(*path_parts) + path_parts = path_parts.map { |p| p.delete_prefix("/").delete_suffix("/") } + normalized_repo_path = Pathname.new(path_parts.join("/")).cleanpath.to_path.delete_prefix("/") + full_path = Pathname.new(File.join(repo_contents_path, normalized_repo_path)).cleanpath.to_path + return unless File.exist?(full_path) - sig { params(project_path: String, relative_file_path: String).returns(T.nilable(String)) } - def dependency_file_path_from_project_path(project_path, relative_file_path) - project_directory = File.dirname(project_path) - dependency_file_path_from_repo_path(project_directory, relative_file_path) + normalized_repo_path = "/#{normalized_repo_path}" unless normalized_repo_path.start_with?("/") + normalized_repo_path + end + + sig { params(project_path: String, relative_file_path: String).returns(T.nilable(String)) } + def dependency_file_path_from_project_path(project_path, relative_file_path) + project_directory = File.dirname(project_path) + dependency_file_path_from_repo_path(project_directory, relative_file_path) + end end end end diff --git a/nuget/lib/dependabot/nuget/native_update_checker/native_update_checker.rb b/nuget/lib/dependabot/nuget/native_update_checker/native_update_checker.rb index 10fe1034c5..32ab30da5c 100644 --- a/nuget/lib/dependabot/nuget/native_update_checker/native_update_checker.rb +++ b/nuget/lib/dependabot/nuget/native_update_checker/native_update_checker.rb @@ -97,7 +97,7 @@ def dependency_file_path sig { returns(T::Array[String]) } def dependency_file_paths dependency_files.map do |file| - NativeDiscoveryJsonReader.dependency_file_path( + Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.dependency_file_path( repo_contents_path: T.must(repo_contents_path), dependency_file: file ) @@ -106,7 +106,7 @@ def dependency_file_paths sig { returns(AnalysisJsonReader) } def request_analysis - discovery_file_path = NativeDiscoveryJsonReader.get_discovery_json_path_for_dependency_file_paths( + discovery_file_path = Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.get_discovery_json_path_for_dependency_file_paths( dependency_file_paths ) analysis_folder_path = AnalysisJsonReader.temp_directory @@ -154,7 +154,7 @@ def write_dependency_info sig { returns(Dependabot::FileParsers::Base::DependencySet) } def discovered_dependencies - NativeDiscoveryJsonReader.load_discovery_for_dependency_file_paths(dependency_file_paths).dependency_set + Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.load_discovery_for_dependency_file_paths(dependency_file_paths).dependency_set end sig { override.returns(T::Boolean) } From 1b3640ae185d09371a425d35ad08027d5a2f16a6 Mon Sep 17 00:00:00 2001 From: sachin-sandhu Date: Wed, 11 Dec 2024 10:41:29 -0500 Subject: [PATCH 11/11] Revert "new commit to test sorbet fixes" This reverts commit 56ba52fc1383a7e2b41915c1e0619b288017e610. --- nuget/lib/dependabot/nuget/file_fetcher.rb | 2 +- nuget/lib/dependabot/nuget/file_parser.rb | 4 +- nuget/lib/dependabot/nuget/file_updater.rb | 12 +- .../native_discovery_json_reader.rb | 420 +++++++++--------- .../native_update_checker.rb | 6 +- 5 files changed, 220 insertions(+), 224 deletions(-) diff --git a/nuget/lib/dependabot/nuget/file_fetcher.rb b/nuget/lib/dependabot/nuget/file_fetcher.rb index 3ad640b7b2..5dacfd983e 100644 --- a/nuget/lib/dependabot/nuget/file_fetcher.rb +++ b/nuget/lib/dependabot/nuget/file_fetcher.rb @@ -26,7 +26,7 @@ def self.required_files_message sig { override.returns(T::Array[DependencyFile]) } def fetch_files - discovery_json_reader = Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.run_discovery_in_directory( + discovery_json_reader = NativeDiscoveryJsonReader.run_discovery_in_directory( repo_contents_path: T.must(repo_contents_path), directory: directory, credentials: credentials diff --git a/nuget/lib/dependabot/nuget/file_parser.rb b/nuget/lib/dependabot/nuget/file_parser.rb index e3adf0c561..c68b06376d 100644 --- a/nuget/lib/dependabot/nuget/file_parser.rb +++ b/nuget/lib/dependabot/nuget/file_parser.rb @@ -41,7 +41,7 @@ def ecosystem def content_json @content_json ||= T.let(begin directory = source&.directory || "/" - discovery_json_reader = Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.run_discovery_in_directory( + discovery_json_reader = Dependabot::Nuget::NativeDiscoveryJsonReader.run_discovery_in_directory( repo_contents_path: T.must(repo_contents_path), directory: directory, credentials: credentials @@ -54,7 +54,7 @@ def content_json def dependencies @dependencies ||= T.let(begin directory = source&.directory || "/" - discovery_json_reader = Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.run_discovery_in_directory( + discovery_json_reader = NativeDiscoveryJsonReader.run_discovery_in_directory( repo_contents_path: T.must(repo_contents_path), directory: directory, credentials: credentials diff --git a/nuget/lib/dependabot/nuget/file_updater.rb b/nuget/lib/dependabot/nuget/file_updater.rb index 43be9c529b..39393b9ae5 100644 --- a/nuget/lib/dependabot/nuget/file_updater.rb +++ b/nuget/lib/dependabot/nuget/file_updater.rb @@ -57,7 +57,7 @@ def updated_dependency_files try_update_projects(dependency) || try_update_json(dependency) end updated_files = dependency_files.filter_map do |f| - dependency_file_path = Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.dependency_file_path( + dependency_file_path = NativeDiscoveryJsonReader.dependency_file_path( repo_contents_path: T.must(repo_contents_path), dependency_file: f ) @@ -97,7 +97,7 @@ def try_update_projects(dependency) # run update for each project file project_files.each do |project_file| project_dependencies = project_dependencies(project_file) - dependency_file_path = Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.dependency_file_path( + dependency_file_path = NativeDiscoveryJsonReader.dependency_file_path( repo_contents_path: T.must(repo_contents_path), dependency_file: project_file ) @@ -128,7 +128,7 @@ def try_update_json(dependency) # We just need to feed the updater a project file, grab the first project_file = T.must(project_files.first) - dependency_file_path = Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.dependency_file_path( + dependency_file_path = NativeDiscoveryJsonReader.dependency_file_path( repo_contents_path: T.must(repo_contents_path), dependency_file: project_file ) @@ -171,10 +171,10 @@ def testonly_update_tooling_calls sig { returns(T.nilable(NativeWorkspaceDiscovery)) } def workspace dependency_file_paths = dependency_files.map do |f| - Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.dependency_file_path(repo_contents_path: T.must(repo_contents_path), - dependency_file: f) + NativeDiscoveryJsonReader.dependency_file_path(repo_contents_path: T.must(repo_contents_path), + dependency_file: f) end - Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.load_discovery_for_dependency_file_paths(dependency_file_paths).workspace_discovery + NativeDiscoveryJsonReader.load_discovery_for_dependency_file_paths(dependency_file_paths).workspace_discovery end sig { params(project_file: Dependabot::DependencyFile).returns(T::Array[String]) } diff --git a/nuget/lib/dependabot/nuget/native_discovery/native_discovery_json_reader.rb b/nuget/lib/dependabot/nuget/native_discovery/native_discovery_json_reader.rb index cad93b4d3a..7bcaadb865 100644 --- a/nuget/lib/dependabot/nuget/native_discovery/native_discovery_json_reader.rb +++ b/nuget/lib/dependabot/nuget/native_discovery/native_discovery_json_reader.rb @@ -10,260 +10,256 @@ module Dependabot module Nuget - module NativeDiscovery - class NativeDiscoveryJsonReader - extend T::Sig + class NativeDiscoveryJsonReader + extend T::Sig - sig { returns(T::Hash[String, NativeDiscoveryJsonReader]) } - def self.cache_directory_to_discovery_json_reader - CacheManager.cache("cache_directory_to_discovery_json_reader") - end + sig { returns(T::Hash[String, NativeDiscoveryJsonReader]) } + def self.cache_directory_to_discovery_json_reader + CacheManager.cache("cache_directory_to_discovery_json_reader") + end - sig { returns(T::Hash[String, NativeDiscoveryJsonReader]) } - def self.cache_dependency_file_paths_to_discovery_json_reader - CacheManager.cache("cache_dependency_file_paths_to_discovery_json_reader") - end + sig { returns(T::Hash[String, NativeDiscoveryJsonReader]) } + def self.cache_dependency_file_paths_to_discovery_json_reader + CacheManager.cache("cache_dependency_file_paths_to_discovery_json_reader") + end - sig { returns(T::Hash[String, String]) } - def self.cache_dependency_file_paths_to_discovery_json_path - CacheManager.cache("cache_dependency_file_paths_to_discovery_json_path") - end + sig { returns(T::Hash[String, String]) } + def self.cache_dependency_file_paths_to_discovery_json_path + CacheManager.cache("cache_dependency_file_paths_to_discovery_json_path") + end - sig { void } - def self.testonly_clear_caches - cache_directory_to_discovery_json_reader.clear - cache_dependency_file_paths_to_discovery_json_reader.clear - cache_dependency_file_paths_to_discovery_json_path.clear - end + sig { void } + def self.testonly_clear_caches + cache_directory_to_discovery_json_reader.clear + cache_dependency_file_paths_to_discovery_json_reader.clear + cache_dependency_file_paths_to_discovery_json_path.clear + end - sig { void } - def self.testonly_clear_discovery_files - # this will get recreated when necessary - FileUtils.rm_rf(discovery_directory) - end + sig { void } + def self.testonly_clear_discovery_files + # this will get recreated when necessary + FileUtils.rm_rf(discovery_directory) + end - # Runs NuGet dependency discovery in the given directory and returns a new instance of NativeDiscoveryJsonReader. - # The location of the resultant JSON file is saved. - sig do - params( - repo_contents_path: String, - directory: String, - credentials: T::Array[Dependabot::Credential] - ).returns(NativeDiscoveryJsonReader) - end - def self.run_discovery_in_directory(repo_contents_path:, directory:, credentials:) - # run discovery - job_file_path = ENV.fetch("DEPENDABOT_JOB_PATH") - discovery_json_path = discovery_file_path_from_workspace_path(directory) - unless File.exist?(discovery_json_path) - NativeHelpers.run_nuget_discover_tool(job_path: job_file_path, - repo_root: repo_contents_path, - workspace_path: directory, - output_path: discovery_json_path, - credentials: credentials) - - Dependabot.logger.info("Discovery JSON content: #{File.read(discovery_json_path)}") - end - load_discovery_for_directory(repo_contents_path: repo_contents_path, directory: directory) + # Runs NuGet dependency discovery in the given directory and returns a new instance of NativeDiscoveryJsonReader. + # The location of the resultant JSON file is saved. + sig do + params( + repo_contents_path: String, + directory: String, + credentials: T::Array[Dependabot::Credential] + ).returns(NativeDiscoveryJsonReader) + end + def self.run_discovery_in_directory(repo_contents_path:, directory:, credentials:) + # run discovery + job_file_path = ENV.fetch("DEPENDABOT_JOB_PATH") + discovery_json_path = discovery_file_path_from_workspace_path(directory) + unless File.exist?(discovery_json_path) + NativeHelpers.run_nuget_discover_tool(job_path: job_file_path, + repo_root: repo_contents_path, + workspace_path: directory, + output_path: discovery_json_path, + credentials: credentials) + + Dependabot.logger.info("Discovery JSON content: #{File.read(discovery_json_path)}") end + load_discovery_for_directory(repo_contents_path: repo_contents_path, directory: directory) + end - # Loads NuGet dependency discovery for the given directory and returns a new instance of - # NativeDiscoveryJsonReader and caches the resultant object. - sig { params(repo_contents_path: String, directory: String).returns(NativeDiscoveryJsonReader) } - def self.load_discovery_for_directory(repo_contents_path:, directory:) - cache_directory_to_discovery_json_reader[directory] ||= begin - discovery_json_reader = discovery_json_reader(repo_contents_path: repo_contents_path, - workspace_path: directory) - cache_directory_to_discovery_json_reader[directory] = discovery_json_reader - dependency_file_cache_key = cache_key_from_dependency_file_paths(discovery_json_reader.dependency_file_paths) - cache_dependency_file_paths_to_discovery_json_reader[dependency_file_cache_key] = discovery_json_reader - discovery_file_path = discovery_file_path_from_workspace_path(directory) - cache_dependency_file_paths_to_discovery_json_path[dependency_file_cache_key] = discovery_file_path - - discovery_json_reader - end + # Loads NuGet dependency discovery for the given directory and returns a new instance of + # NativeDiscoveryJsonReader and caches the resultant object. + sig { params(repo_contents_path: String, directory: String).returns(NativeDiscoveryJsonReader) } + def self.load_discovery_for_directory(repo_contents_path:, directory:) + cache_directory_to_discovery_json_reader[directory] ||= begin + discovery_json_reader = discovery_json_reader(repo_contents_path: repo_contents_path, + workspace_path: directory) + cache_directory_to_discovery_json_reader[directory] = discovery_json_reader + dependency_file_cache_key = cache_key_from_dependency_file_paths(discovery_json_reader.dependency_file_paths) + cache_dependency_file_paths_to_discovery_json_reader[dependency_file_cache_key] = discovery_json_reader + discovery_file_path = discovery_file_path_from_workspace_path(directory) + cache_dependency_file_paths_to_discovery_json_path[dependency_file_cache_key] = discovery_file_path + + discovery_json_reader end + end - # Retrieves the cached NativeDiscoveryJsonReader object for the given dependency file paths. - sig { params(dependency_file_paths: T::Array[String]).returns(NativeDiscoveryJsonReader) } - def self.load_discovery_for_dependency_file_paths(dependency_file_paths) - dependency_file_cache_key = cache_key_from_dependency_file_paths(dependency_file_paths) - T.must(cache_dependency_file_paths_to_discovery_json_reader[dependency_file_cache_key]) - end + # Retrieves the cached NativeDiscoveryJsonReader object for the given dependency file paths. + sig { params(dependency_file_paths: T::Array[String]).returns(NativeDiscoveryJsonReader) } + def self.load_discovery_for_dependency_file_paths(dependency_file_paths) + dependency_file_cache_key = cache_key_from_dependency_file_paths(dependency_file_paths) + T.must(cache_dependency_file_paths_to_discovery_json_reader[dependency_file_cache_key]) + end - # Retrieves the cached location of the discovery JSON file for the given dependency file paths. - sig { params(dependency_file_paths: T::Array[String]).returns(String) } - def self.get_discovery_json_path_for_dependency_file_paths(dependency_file_paths) - dependency_file_cache_key = cache_key_from_dependency_file_paths(dependency_file_paths) - T.must(cache_dependency_file_paths_to_discovery_json_path[dependency_file_cache_key]) - end + # Retrieves the cached location of the discovery JSON file for the given dependency file paths. + sig { params(dependency_file_paths: T::Array[String]).returns(String) } + def self.get_discovery_json_path_for_dependency_file_paths(dependency_file_paths) + dependency_file_cache_key = cache_key_from_dependency_file_paths(dependency_file_paths) + T.must(cache_dependency_file_paths_to_discovery_json_path[dependency_file_cache_key]) + end - sig { params(repo_contents_path: String, dependency_file: Dependabot::DependencyFile).returns(String) } - def self.dependency_file_path(repo_contents_path:, dependency_file:) - dep_file_path = Pathname.new(File.join(dependency_file.directory, dependency_file.name)).cleanpath.to_path - dep_file_path.delete_prefix("#{repo_contents_path}/") - end + sig { params(repo_contents_path: String, dependency_file: Dependabot::DependencyFile).returns(String) } + def self.dependency_file_path(repo_contents_path:, dependency_file:) + dep_file_path = Pathname.new(File.join(dependency_file.directory, dependency_file.name)).cleanpath.to_path + dep_file_path.delete_prefix("#{repo_contents_path}/") + end + + sig { returns(String) } + def self.discovery_map_file_path + File.join(discovery_directory, "discovery_map.json") + end - sig { returns(String) } - def self.discovery_map_file_path - File.join(discovery_directory, "discovery_map.json") + sig { params(workspace_path: String).returns(String) } + def self.discovery_file_path_from_workspace_path(workspace_path) + # Given an update directory (also known as a workspace path), this function returns the path where the discovery + # JSON file is located. This function is called both by methods that need to write the discovery JSON file and + # by methods that need to read the discovery JSON file. This function is also called by multiple processes so + # we need a way to retain the data. This is accomplished by the following steps: + # 1. Check a well-known file for a mapping of workspace_path => discovery file path. If found, return it. + # 2. If the path is not found, generate a new path, save it to the well-known file, and return the value. + discovery_map_contents = File.exist?(discovery_map_file_path) ? File.read(discovery_map_file_path) : "{}" + discovery_map = T.let(JSON.parse(discovery_map_contents), T::Hash[String, String]) + + discovery_json_path = discovery_map[workspace_path] + if discovery_json_path + Dependabot.logger.info("Discovery JSON path for workspace path [#{workspace_path}] found in file " \ + "[#{discovery_map_file_path}] at location [#{discovery_json_path}]") + return discovery_json_path end - sig { params(workspace_path: String).returns(String) } - def self.discovery_file_path_from_workspace_path(workspace_path) - # Given an update directory (also known as a workspace path), this function returns the path where the discovery - # JSON file is located. This function is called both by methods that need to write the discovery JSON file and - # by methods that need to read the discovery JSON file. This function is also called by multiple processes so - # we need a way to retain the data. This is accomplished by the following steps: - # 1. Check a well-known file for a mapping of workspace_path => discovery file path. If found, return it. - # 2. If the path is not found, generate a new path, save it to the well-known file, and return the value. - discovery_map_contents = File.exist?(discovery_map_file_path) ? File.read(discovery_map_file_path) : "{}" - discovery_map = T.let(JSON.parse(discovery_map_contents), T::Hash[String, String]) - - discovery_json_path = discovery_map[workspace_path] - if discovery_json_path - Dependabot.logger.info("Discovery JSON path for workspace path [#{workspace_path}] found in file " \ - "[#{discovery_map_file_path}] at location [#{discovery_json_path}]") - return discovery_json_path - end + # no discovery JSON path found; generate a new one, but first find a suitable location + discovery_json_counter = 1 + new_discovery_json_path = "" + loop do + new_discovery_json_path = File.join(discovery_directory, "discovery.#{discovery_json_counter}.json") + break unless File.exist?(new_discovery_json_path) + + discovery_json_counter += 1 + end - # no discovery JSON path found; generate a new one, but first find a suitable location - discovery_json_counter = 1 - new_discovery_json_path = "" - loop do - new_discovery_json_path = File.join(discovery_directory, "discovery.#{discovery_json_counter}.json") - break unless File.exist?(new_discovery_json_path) + discovery_map[workspace_path] = new_discovery_json_path - discovery_json_counter += 1 - end + File.write(discovery_map_file_path, discovery_map.to_json) + Dependabot.logger.info("Discovery JSON path for workspace path [#{workspace_path}] created for file " \ + "[#{discovery_map_file_path}] at location [#{new_discovery_json_path}]") + new_discovery_json_path + end - discovery_map[workspace_path] = new_discovery_json_path + sig { params(dependency_file_paths: T::Array[String]).returns(String) } + def self.cache_key_from_dependency_file_paths(dependency_file_paths) + dependency_file_paths.sort.join(",") + end - File.write(discovery_map_file_path, discovery_map.to_json) - Dependabot.logger.info("Discovery JSON path for workspace path [#{workspace_path}] created for file " \ - "[#{discovery_map_file_path}] at location [#{new_discovery_json_path}]") - new_discovery_json_path - end + sig { returns(String) } + def self.discovery_directory + t = File.join(Dir.home, ".dependabot") + FileUtils.mkdir_p(t) + t + end - sig { params(dependency_file_paths: T::Array[String]).returns(String) } - def self.cache_key_from_dependency_file_paths(dependency_file_paths) - dependency_file_paths.sort.join(",") - end + sig { params(repo_contents_path: String, workspace_path: String).returns(NativeDiscoveryJsonReader) } + def self.discovery_json_reader(repo_contents_path:, workspace_path:) + discovery_file_path = discovery_file_path_from_workspace_path(workspace_path) + discovery_json = DependencyFile.new( + name: Pathname.new(discovery_file_path).cleanpath.to_path, + directory: discovery_directory, + type: "file", + content: File.read(discovery_file_path) + ) + NativeDiscoveryJsonReader.new(repo_contents_path: repo_contents_path, discovery_json: discovery_json) + end - sig { returns(String) } - def self.discovery_directory - t = File.join(Dir.home, ".dependabot") - FileUtils.mkdir_p(t) - t - end + sig { returns(T.nilable(NativeWorkspaceDiscovery)) } + attr_reader :workspace_discovery - sig { params(repo_contents_path: String, workspace_path: String).returns(NativeDiscoveryJsonReader) } - def self.discovery_json_reader(repo_contents_path:, workspace_path:) - discovery_file_path = discovery_file_path_from_workspace_path(workspace_path) - discovery_json = DependencyFile.new( - name: Pathname.new(discovery_file_path).cleanpath.to_path, - directory: discovery_directory, - type: "file", - content: File.read(discovery_file_path) - ) - NativeDiscoveryJsonReader.new(repo_contents_path: repo_contents_path, discovery_json: discovery_json) - end + sig { returns(Dependabot::FileParsers::Base::DependencySet) } + attr_reader :dependency_set - sig { returns(T.nilable(NativeWorkspaceDiscovery)) } - attr_reader :workspace_discovery + sig { returns(T::Array[String]) } + attr_reader :dependency_file_paths - sig { returns(Dependabot::FileParsers::Base::DependencySet) } - attr_reader :dependency_set + sig { params(repo_contents_path: String, discovery_json: DependencyFile).void } + def initialize(repo_contents_path:, discovery_json:) + @repo_contents_path = repo_contents_path + @discovery_json = discovery_json + @workspace_discovery = T.let(read_workspace_discovery, T.nilable(Dependabot::Nuget::NativeWorkspaceDiscovery)) + @dependency_set = T.let(read_dependency_set, Dependabot::FileParsers::Base::DependencySet) + @dependency_file_paths = T.let(read_dependency_file_paths, T::Array[String]) + end - sig { returns(T::Array[String]) } - attr_reader :dependency_file_paths + private - sig { params(repo_contents_path: String, discovery_json: DependencyFile).void } - def initialize(repo_contents_path:, discovery_json:) - @repo_contents_path = repo_contents_path - @discovery_json = discovery_json - @workspace_discovery = T.let(read_workspace_discovery, T.nilable(Dependabot::Nuget::NativeWorkspaceDiscovery)) - @dependency_set = T.let(read_dependency_set, Dependabot::FileParsers::Base::DependencySet) - @dependency_file_paths = T.let(read_dependency_file_paths, T::Array[String]) - end + sig { returns(String) } + attr_reader :repo_contents_path - private + sig { returns(DependencyFile) } + attr_reader :discovery_json - sig { returns(String) } - attr_reader :repo_contents_path + sig { returns(T.nilable(NativeWorkspaceDiscovery)) } + def read_workspace_discovery + return nil unless discovery_json.content - sig { returns(DependencyFile) } - attr_reader :discovery_json + parsed_json = T.let(JSON.parse(T.must(discovery_json.content)), T::Hash[String, T.untyped]) + NativeWorkspaceDiscovery.from_json(parsed_json) + rescue JSON::ParserError + raise Dependabot::DependencyFileNotParseable, discovery_json.path + end - sig { returns(T.nilable(NativeWorkspaceDiscovery)) } - def read_workspace_discovery - return nil unless discovery_json.content + sig { returns(Dependabot::FileParsers::Base::DependencySet) } + def read_dependency_set + dependency_set = Dependabot::FileParsers::Base::DependencySet.new + return dependency_set unless workspace_discovery - parsed_json = T.let(JSON.parse(T.must(discovery_json.content)), T::Hash[String, T.untyped]) - NativeWorkspaceDiscovery.from_json(parsed_json) - rescue JSON::ParserError - raise Dependabot::DependencyFileNotParseable, discovery_json.path + workspace_result = T.must(workspace_discovery) + workspace_result.projects.each do |project| + dependency_set += project.dependency_set end + if workspace_result.dotnet_tools_json + dependency_set += T.must(workspace_result.dotnet_tools_json).dependency_set + end + dependency_set += T.must(workspace_result.global_json).dependency_set if workspace_result.global_json - sig { returns(Dependabot::FileParsers::Base::DependencySet) } - def read_dependency_set - dependency_set = Dependabot::FileParsers::Base::DependencySet.new - return dependency_set unless workspace_discovery - - workspace_result = T.must(workspace_discovery) - workspace_result.projects.each do |project| - dependency_set += project.dependency_set - end - if workspace_result.dotnet_tools_json - dependency_set += T.must(workspace_result.dotnet_tools_json).dependency_set - end - dependency_set += T.must(workspace_result.global_json).dependency_set if workspace_result.global_json + dependency_set + end - dependency_set + sig { returns(T::Array[String]) } + def read_dependency_file_paths + dependency_file_paths = T.let([], T::Array[T.nilable(String)]) + dependency_file_paths << dependency_file_path_from_repo_path("global.json") if workspace_discovery&.global_json + if workspace_discovery&.dotnet_tools_json + dependency_file_paths << dependency_file_path_from_repo_path(".config/dotnet-tools.json") end - sig { returns(T::Array[String]) } - def read_dependency_file_paths - dependency_file_paths = T.let([], T::Array[T.nilable(String)]) - if workspace_discovery&.global_json - dependency_file_paths << dependency_file_path_from_repo_path("global.json") - end - if workspace_discovery&.dotnet_tools_json - dependency_file_paths << dependency_file_path_from_repo_path(".config/dotnet-tools.json") + projects = workspace_discovery&.projects || [] + projects.each do |project| + dependency_file_paths << dependency_file_path_from_repo_path(project.file_path) + dependency_file_paths += project.imported_files.map do |f| + dependency_file_path_from_project_path(project.file_path, f) end - - projects = workspace_discovery&.projects || [] - projects.each do |project| - dependency_file_paths << dependency_file_path_from_repo_path(project.file_path) - dependency_file_paths += project.imported_files.map do |f| - dependency_file_path_from_project_path(project.file_path, f) - end - dependency_file_paths += project.additional_files.map do |f| - dependency_file_path_from_project_path(project.file_path, f) - end + dependency_file_paths += project.additional_files.map do |f| + dependency_file_path_from_project_path(project.file_path, f) end - - deduped_dependency_file_paths = T.let(Set.new(dependency_file_paths.compact), T::Set[String]) - result = deduped_dependency_file_paths.sort - result end - sig { params(path_parts: String).returns(T.nilable(String)) } - def dependency_file_path_from_repo_path(*path_parts) - path_parts = path_parts.map { |p| p.delete_prefix("/").delete_suffix("/") } - normalized_repo_path = Pathname.new(path_parts.join("/")).cleanpath.to_path.delete_prefix("/") - full_path = Pathname.new(File.join(repo_contents_path, normalized_repo_path)).cleanpath.to_path - return unless File.exist?(full_path) + deduped_dependency_file_paths = T.let(Set.new(dependency_file_paths.compact), T::Set[String]) + result = deduped_dependency_file_paths.sort + result + end - normalized_repo_path = "/#{normalized_repo_path}" unless normalized_repo_path.start_with?("/") - normalized_repo_path - end + sig { params(path_parts: String).returns(T.nilable(String)) } + def dependency_file_path_from_repo_path(*path_parts) + path_parts = path_parts.map { |p| p.delete_prefix("/").delete_suffix("/") } + normalized_repo_path = Pathname.new(path_parts.join("/")).cleanpath.to_path.delete_prefix("/") + full_path = Pathname.new(File.join(repo_contents_path, normalized_repo_path)).cleanpath.to_path + return unless File.exist?(full_path) - sig { params(project_path: String, relative_file_path: String).returns(T.nilable(String)) } - def dependency_file_path_from_project_path(project_path, relative_file_path) - project_directory = File.dirname(project_path) - dependency_file_path_from_repo_path(project_directory, relative_file_path) - end + normalized_repo_path = "/#{normalized_repo_path}" unless normalized_repo_path.start_with?("/") + normalized_repo_path + end + + sig { params(project_path: String, relative_file_path: String).returns(T.nilable(String)) } + def dependency_file_path_from_project_path(project_path, relative_file_path) + project_directory = File.dirname(project_path) + dependency_file_path_from_repo_path(project_directory, relative_file_path) end end end diff --git a/nuget/lib/dependabot/nuget/native_update_checker/native_update_checker.rb b/nuget/lib/dependabot/nuget/native_update_checker/native_update_checker.rb index 32ab30da5c..10fe1034c5 100644 --- a/nuget/lib/dependabot/nuget/native_update_checker/native_update_checker.rb +++ b/nuget/lib/dependabot/nuget/native_update_checker/native_update_checker.rb @@ -97,7 +97,7 @@ def dependency_file_path sig { returns(T::Array[String]) } def dependency_file_paths dependency_files.map do |file| - Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.dependency_file_path( + NativeDiscoveryJsonReader.dependency_file_path( repo_contents_path: T.must(repo_contents_path), dependency_file: file ) @@ -106,7 +106,7 @@ def dependency_file_paths sig { returns(AnalysisJsonReader) } def request_analysis - discovery_file_path = Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.get_discovery_json_path_for_dependency_file_paths( + discovery_file_path = NativeDiscoveryJsonReader.get_discovery_json_path_for_dependency_file_paths( dependency_file_paths ) analysis_folder_path = AnalysisJsonReader.temp_directory @@ -154,7 +154,7 @@ def write_dependency_info sig { returns(Dependabot::FileParsers::Base::DependencySet) } def discovered_dependencies - Dependabot::Nuget::NativeDiscovery::NativeDiscoveryJsonReader.load_discovery_for_dependency_file_paths(dependency_file_paths).dependency_set + NativeDiscoveryJsonReader.load_discovery_for_dependency_file_paths(dependency_file_paths).dependency_set end sig { override.returns(T::Boolean) }