From 9ff4edbe50fc565a9187936e7df8a1bef841e68a Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 5 Apr 2023 15:27:14 +0200 Subject: [PATCH 1/2] Bump version to 0.9.8-beta1 Signed-off-by: Philippe Ombredanne --- CHANGELOG.rst | 15 +++++++++++++++ build.sh | 2 +- release.sh | 2 +- setup.cfg | 2 +- src/nuget-inspector/nuget-inspector.csproj | 6 +++--- 5 files changed, 21 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 91aaab1a..b8fdfe6a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,21 @@ Changelog ========= +v0.9.8 +------- + +This is a major feature update release with these updates and API breaking changes: + +* Add new "--with-nuget-org" command line option to optionally include the + nuget.org API feeds to the set of nuget.config-provided endpoints. Note that + this is always included if there is no provided or available nuget.config + +* Add extra debug tracing with the --debug option + +* Do not create API or download URLs using a default. Only use API-provided values. + +* Ensure we do not fail with a Package reference with no version. + v0.9.7 ------- diff --git a/build.sh b/build.sh index 1909fecf..c61fe0cf 100755 --- a/build.sh +++ b/build.sh @@ -17,6 +17,6 @@ dotnet publish \ --runtime linux-x64 \ --self-contained true \ --configuration Release \ - -p:Version=0.9.7 \ + -p:Version=0.9.8-beta1 \ --output build \ src/nuget-inspector/nuget-inspector.csproj diff --git a/release.sh b/release.sh index 9dc0f50b..304ad7ff 100755 --- a/release.sh +++ b/release.sh @@ -16,7 +16,7 @@ rm -rf release/ mkdir release -VERSION=0.9.7 +VERSION=0.9.8-beta1 TARGET_BASE=nuget-inspector-$(git describe) diff --git a/setup.cfg b/setup.cfg index a37c064f..fe979612 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,7 +1,7 @@ [bumpversion] commit = False tag = False -current_version = 0.9.7 +current_version = 0.9.8-beta1 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+))? serialize = {major}.{minor}.{patch}-{release} diff --git a/src/nuget-inspector/nuget-inspector.csproj b/src/nuget-inspector/nuget-inspector.csproj index 63cecb45..2983445f 100644 --- a/src/nuget-inspector/nuget-inspector.csproj +++ b/src/nuget-inspector/nuget-inspector.csproj @@ -23,11 +23,11 @@ nuget-inspector nuget-inspector nuget-inspector - 0.9.7 + 0.9.8-beta1 nexB Inc. nexB Inc - 0.9.7.0 - 0.9.7.0 + 0.9.8.0 + 0.9.8.0 A NuGet and Dotnet package dependency resolver https://github.com/nexB/nuget-inspector Apache-2.0 AND MIT From 532d0944a232ce7e0172aa146ae3239845aa811d Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 5 Apr 2023 15:29:21 +0200 Subject: [PATCH 2/2] Add new CLI options * --debug creates extensive debug outputs * --with-nuget-org add nuget.org public endpoints if requested * Also ensure we do not fail with a package ref without a version Signed-off-by: Philippe Ombredanne --- src/nuget-inspector/Config.cs | 3 +- src/nuget-inspector/Models.cs | 3 +- src/nuget-inspector/NugetApi.cs | 121 +++++++++++++++++--------- src/nuget-inspector/Options.cs | 14 +++ src/nuget-inspector/Program.cs | 24 ++++- src/nuget-inspector/ProjectScanner.cs | 12 +-- 6 files changed, 122 insertions(+), 55 deletions(-) diff --git a/src/nuget-inspector/Config.cs b/src/nuget-inspector/Config.cs index 30136324..5cf08ea6 100644 --- a/src/nuget-inspector/Config.cs +++ b/src/nuget-inspector/Config.cs @@ -8,6 +8,7 @@ public static class Config public static bool TRACE_NET = false; public static bool TRACE_DEEP = false; public static bool TRACE_META = false; - public const string NUGET_INSPECTOR_VERSION = "0.9.7"; + public static bool TRACE_OUTPUT = false; + public const string NUGET_INSPECTOR_VERSION = "0.9.8-beta1"; #pragma warning restore CA2211 } \ No newline at end of file diff --git a/src/nuget-inspector/Models.cs b/src/nuget-inspector/Models.cs index 3cc7ebfe..33c20cfd 100644 --- a/src/nuget-inspector/Models.cs +++ b/src/nuget-inspector/Models.cs @@ -442,8 +442,7 @@ public void UpdateAttributes( if ( string.IsNullOrWhiteSpace(api_data_url) && download_url.StartsWith("https://api.nuget.org/") - && !string.IsNullOrWhiteSpace(synthetic_api_data_url) - ) + && !string.IsNullOrWhiteSpace(synthetic_api_data_url)) { api_data_url = synthetic_api_data_url; } diff --git a/src/nuget-inspector/NugetApi.cs b/src/nuget-inspector/NugetApi.cs index c6c9de68..4e52ea9f 100644 --- a/src/nuget-inspector/NugetApi.cs +++ b/src/nuget-inspector/NugetApi.cs @@ -33,7 +33,7 @@ public class NugetApi private readonly Dictionary catalog_entry_by_catalog_url = new(); private readonly Dictionary> psmrs_by_package_name = new(); private readonly Dictionary psmr_by_identity = new(); - private readonly Dictionary<(PackageIdentity, NuGetFramework), PackageDownload> download_by_identity = new(); + private readonly Dictionary<(PackageIdentity, NuGetFramework), PackageDownload?> download_by_identity = new(); private readonly Dictionary<(PackageIdentity, NuGetFramework), SourcePackageDependencyInfo> spdi_by_identity = new(); private readonly List source_repositories = new(); @@ -44,7 +44,11 @@ public class NugetApi private readonly NuGetFramework project_framework; - public NugetApi(string nuget_config_path, string project_root_path, NuGetFramework project_framework) + public NugetApi( + string nuget_config_path, + string project_root_path, + NuGetFramework project_framework, + bool with_nuget_org) { this.project_framework = project_framework; List> providers = new(); @@ -56,7 +60,8 @@ public NugetApi(string nuget_config_path, string project_root_path, NuGetFramewo PopulateResources( providers: providers, - settings: settings); + settings: settings, + with_nuget_org: with_nuget_org); } /// @@ -114,13 +119,13 @@ private List FindPackageVersionsThroughCache( /// PackageSearchMetadataRegistration or null public PackageSearchMetadataRegistration? FindPackageVersion(PackageIdentity pid) { - if (Config.TRACE_NET) + if (Config.TRACE) Console.WriteLine($"FindPackageVersion: {pid}"); if (psmr_by_identity.TryGetValue(key: pid, out PackageSearchMetadataRegistration? psmr)) { - if (Config.TRACE_NET) - Console.WriteLine($"Metadata Cache hit for '{pid}'"); + if (Config.TRACE) + Console.WriteLine($" Metadata Cache hit for '{pid}'"); return psmr; } @@ -139,7 +144,7 @@ private List FindPackageVersionsThroughCache( if (psmr != null) { - if (Config.TRACE_NET) + if (Config.TRACE) Console.WriteLine($" Found metadata for '{pid}' from: {metadata_resources}"); psmr_by_identity[pid] = psmr; return psmr; @@ -147,22 +152,24 @@ private List FindPackageVersionsThroughCache( } catch (Exception ex) { - if (Config.TRACE_NET) - Console.WriteLine($"FAILED to Fetch metadata for '{pid}' with: {ex.StackTrace}"); + if (Config.TRACE) + Console.WriteLine($" FAILED to Fetch metadata for '{pid}' with: {ex.StackTrace}"); exceptions.Add(item: ex); } } - if (Config.TRACE_NET && exceptions.Any()) - { - Console.WriteLine($"ERROR: No package found for {pid}."); - foreach (var ex in exceptions) - Console.WriteLine($" {ex}"); - } + var error_message = $"No package metadata found for {pid}."; + foreach (var ex in exceptions) + error_message += $"\n {ex}"; + + if (Config.TRACE) + Console.WriteLine(error_message); + // cache this null too psmr_by_identity[pid] = null; - return null; + + throw new Exception(error_message); } /// @@ -292,26 +299,34 @@ public static ISettings LoadNugetConfigSettings( /// from a feed URL and a nuget.config file path. /// These are MetadataResourceList and DependencyInfoResourceList attributes. /// - private void PopulateResources(List> providers, ISettings settings) + private void PopulateResources( + List> providers, + ISettings settings, + bool with_nuget_org = false) { PackageSourceProvider package_source_provider = new(settings: settings); List package_sources = package_source_provider.LoadPackageSources().ToList(); if (Config.TRACE) Console.WriteLine($"\nPopulateResources: Loaded {package_sources.Count} package sources from nuget.config"); - // always nuget.org as last resort - var nuget_source = new PackageSource(source: "https://api.nuget.org/v3/index.json", name : "nuget.org"); - package_sources.Add(nuget_source); + if (with_nuget_org || !package_sources.Any()) + { + // Use nuget.org as last resort + var nuget_source = new PackageSource( + source: "https://api.nuget.org/v3/index.json", + name : "nuget.org"); + package_sources.Add(nuget_source); + } HashSet seen = new(); foreach (PackageSource package_source in package_sources) { - var uri = package_source.SourceUri.ToString(); - if (seen.Contains(uri)) + var source_url = package_source.SourceUri.ToString(); + if (seen.Contains(source_url)) continue; SourceRepository source_repository = new(source: package_source, providers: providers); AddSourceRepo(source_repo: source_repository); - seen.Add(uri); + seen.Add(source_url); } } @@ -451,24 +466,18 @@ public IEnumerable GetPackageDependenciesForPackage(PackageId if (spdi != null) { download = PackageDownload.FromSpdi(spdi); + download_by_identity[(identity, project_framework)] = download; } else { - if (Config.TRACE_NET) - Console.WriteLine($" Crafting plain download for package '{identity}'"); + if (Config.TRACE) + Console.WriteLine($" No download info available for package '{identity}'"); - // Last resort we craft a synthetic download URL - string name_lower = identity.Id.ToLower(); - string version_lower = identity.Version.ToString().ToLower(); - download = new() - { - download_url = $"https://api.nuget.org/v3-flatcontainer/{name_lower}/{version_lower}/{name_lower}.{version_lower}.nupkg" - }; + download_by_identity[(identity, project_framework)] = download; } - download_by_identity[(identity, project_framework)] = download; } - if (!with_details || (with_details && download.IsEnhanced())) + if (!with_details || (with_details && download?.IsEnhanced() == true)) return download; // We need to fetch the SHA512 (and the size) @@ -511,10 +520,12 @@ public IEnumerable GetPackageDependenciesForPackage(PackageId } string hash = catalog_entry["packageHash"]!.ToString(); - download.hash = Convert.ToHexString(Convert.FromBase64String(hash)); - download.hash_algorithm = catalog_entry["packageHashAlgorithm"]!.ToString(); - download.size = (int)catalog_entry["packageSize"]!; - + if (download != null) + { + download.hash = Convert.ToHexString(Convert.FromBase64String(hash)); + download.hash_algorithm = catalog_entry["packageHashAlgorithm"]!.ToString(); + download.size = (int)catalog_entry["packageSize"]!; + } if (Config.TRACE_NET) Console.WriteLine($" download: {download}"); download_by_identity[(identity, project_framework)] = download; @@ -656,7 +667,17 @@ public HashSet ResolveDependenciesForPackageReferen var packages = new List(); foreach (var targetref in target_references) - packages.Add(PackageId.FromReference(targetref)); + { + try + { + packages.Add(PackageId.FromReference(targetref)); + } + catch (Exception ex) + { + Console.WriteLine($" FAILED: targetref: {targetref}"); + throw new Exception(targetref.ToString(), ex); + } + } walk_context.ProjectLibraryProviders.Add(new ProjectLibraryProvider(packages)); @@ -913,10 +934,26 @@ public override string ToString() public static PackageId FromReference(PackageReference reference) { - bool allow_prerel = reference.HasAllowedVersions && reference.AllowedVersions.MinVersion.IsPrerelease; + if (reference == null) + throw new ArgumentNullException(nameof(reference)); + var av = reference.AllowedVersions; + bool allow_prerel = false; + string? version = null; + if (av != null) + { + var mv = reference.AllowedVersions.MinVersion; + if (mv != null) + allow_prerel = mv.IsPrerelease; + version = reference.AllowedVersions.ToNormalizedString(); + } + if (version == null && reference.PackageIdentity.Version != null) + { + version = reference.PackageIdentity.Version.ToString(); + } + return new PackageId( - id:reference.PackageIdentity.Id, - version: reference.AllowedVersions.ToNormalizedString(), + id: reference.PackageIdentity.Id, + version: version ?? "", allow_prerelease_versions: allow_prerel ); } diff --git a/src/nuget-inspector/Options.cs b/src/nuget-inspector/Options.cs index 4eb44936..3d647116 100644 --- a/src/nuget-inspector/Options.cs +++ b/src/nuget-inspector/Options.cs @@ -26,10 +26,13 @@ public class Options description: "Path to a nuget.config file to use, ignoring all other nuget.config.")] public string NugetConfigPath = ""; + // If True, return extra metadata details when available such as SHA512 public bool WithDetails; public bool WithFallback; + public bool WithNuGetOrg; public bool ShowHelp; public bool Verbose; + public bool Debug; public bool ShowVersion; public bool ShowAbout; @@ -63,12 +66,18 @@ public List AsCliList() if (Verbose) options.Add("--verbose"); + if (Debug) + options.Add("--debug"); + if (WithDetails) options.Add("--with-details"); if (WithFallback) options.Add("--with-fallback"); + if (WithNuGetOrg) + options.Add("--with-nuget-org"); + return options; } @@ -96,10 +105,15 @@ public List AsCliList() command_options.Add(prototype: "with-fallback", description: "Optionally use a plain XML project file parser as fallback from failures.", action: value => options.WithDetails = value != null); + command_options.Add(prototype: "with-nuget-org", description: "Optionally use the officila, public nuget.org API as a fallback in addition to nuget.config-configured API sources.", + action: value => options.WithNuGetOrg = value != null); + command_options.Add(prototype: "h|help", description: "Show this message and exit.", action: value => options.ShowHelp = value != null); command_options.Add(prototype: "v|verbose", description: "Display more verbose output.", action: value => options.Verbose = value != null); + command_options.Add(prototype: "debug", description: "Display very verbose debug output.", + action: value => options.Debug = value != null); command_options.Add(prototype: "version", description: "Display nuget-inspector version and exit.", action: value => options.ShowVersion = value != null); command_options.Add(prototype: "about", description: "Display information about nuget-inspector and exit.", diff --git a/src/nuget-inspector/Program.cs b/src/nuget-inspector/Program.cs index 97cc9e9a..106b189c 100644 --- a/src/nuget-inspector/Program.cs +++ b/src/nuget-inspector/Program.cs @@ -1,6 +1,6 @@ using System.Diagnostics; -using System.Xml; using Microsoft.Build.Locator; +using Newtonsoft.Json; using NuGet.Frameworks; namespace NugetInspector; @@ -99,7 +99,8 @@ private static ExecutionResult ExecuteInspector(Options options) var nuget_api_service = new NugetApi( nuget_config_path: options.NugetConfigPath, project_root_path: project_options.ProjectDirectory, - project_framework: project_framework); + project_framework: project_framework, + with_nuget_org: options.WithNuGetOrg); var scanner = new ProjectScanner( options: project_options, @@ -134,6 +135,17 @@ private static ExecutionResult ExecuteInspector(Options options) var output_formatter = new OutputFormatJson(scan_result: scan_result); output_formatter.Write(); + if (Config.TRACE_OUTPUT) + { + Console.WriteLine("\n=============JSON OUTPUT================"); + string output = JsonConvert.SerializeObject( + value: output_formatter.scan_output, + formatting: Formatting.Indented); + Console.WriteLine(output); + + Console.WriteLine("=======================================\n"); + } + BasePackage project_package = scan_result.project_package; bool success = scan_result.Status == ScanResult.ResultStatus.Success; @@ -246,6 +258,14 @@ private static ParsedOptions ParseCliArgs(string[] args) { Config.TRACE = true; } + if (options.Debug) + { + Config.TRACE = true; + Config.TRACE_DEEP = true; + Config.TRACE_META = true; + Config.TRACE_NET = true; + Config.TRACE_OUTPUT = true; + } if (Config.TRACE_ARGS) Console.WriteLine($"argument: with-details: {options.WithDetails}"); diff --git a/src/nuget-inspector/ProjectScanner.cs b/src/nuget-inspector/ProjectScanner.cs index 239f6329..03f5344c 100644 --- a/src/nuget-inspector/ProjectScanner.cs +++ b/src/nuget-inspector/ProjectScanner.cs @@ -131,9 +131,11 @@ public void FetchDependenciesMetadata(ScanResult scan_result, bool with_details { if (Config.TRACE_META) Console.WriteLine($"\nFetchDependenciesMetadata: with_details: {with_details}"); + foreach (BasePackage dep in scan_result.project_package.dependencies) { dep.Update(nugetApi: NugetApiService, with_details: with_details); + if (Config.TRACE_META) Console.WriteLine($" Fetched for {dep.name}@{dep.version}"); } @@ -159,14 +161,6 @@ public ScanResult RunScan() project_package = project }; - // (string? framework_warning, NuGetFramework project_framework) = FrameworkFinder.GetFramework( - // RequestedFramework: Options.TargetFramework, - // ProjectFilePath: Options.ProjectFilePath - // ); - // if (framework_warning != null) - // scan_result.warnings.Add(framework_warning); - // Options.ProjectFramework = project_framework.GetShortFolderName(); - /* * Try each data file in sequence to resolve packages for a project: * 1. start with modern lockfiles such as project-assets.json and older projects.json.lock @@ -178,7 +172,9 @@ public ScanResult RunScan() DependencyResolution resolution; IDependencyProcessor resolver; + // project.assets.json is the gold standard when available + // TODO: make the use of lockfiles optional if (FileExists(path: ScannerOptions.ProjectAssetsJsonPath!)) { if (Config.TRACE)