fix(java): update logic to detect pom.xml file snapshot artifacts from remote repositories (aquasecurity#6412)

Author: DmitriyLewen

docs/docs/coverage/language/java.md

@@ -42,7 +42,19 @@ Trivy parses your `pom.xml` file and tries to find files with dependencies from
 - relativePath field[^5]
 - local repository directory[^6].
 
-If your machine doesn't have the necessary files - Trivy tries to find the information about these dependencies in the [maven repository](https://repo.maven.apache.org/maven2/).
+### remote repositories
+If your machine doesn't have the necessary files - Trivy tries to find the information about these dependencies in the remote repositories:
+
+- [repositories from pom files][maven-pom-repos]
+- [maven central repository][maven-central]
+
+Trivy reproduces Maven's repository selection and priority:
+
+- for snapshot artifacts:
+    - check only snapshot repositories from pom files (if exists)
+- for other artifacts:
+    - check release repositories from pom files (if exists)
+    - check [maven central][maven-central]
 
 !!! Note
     Trivy only takes information about packages. We don't take a list of vulnerabilities for packages from the `maven repository`.
@@ -92,4 +104,6 @@ Make sure that you have cache[^8] directory to find licenses from `*.pom` depend
 [^8]: The supported directories are `$GRADLE_USER_HOME/caches` and `$HOME/.gradle/caches` (`%HOMEPATH%\.gradle\caches` for Windows).
 
 [dependency-graph]: ../../configuration/reporting.md#show-origins-of-vulnerable-dependencies
-[maven-invoker-plugin]: https://maven.apache.org/plugins/maven-invoker-plugin/usage.html
+[maven-invoker-plugin]: https://maven.apache.org/plugins/maven-invoker-plugin/usage.html
+[maven-central]: https://repo.maven.apache.org/maven2/
+[maven-pom-repos]: https://maven.apache.org/settings.html#repositories

pkg/dependency/parser/java/pom/parse.go

@@ -30,8 +30,9 @@ const (
 )
 
 type options struct {
-	offline     bool
-	remoteRepos []string
+	offline             bool
+	releaseRemoteRepos  []string
+	snapshotRemoteRepos []string
 }
 
 type option func(*options)
@@ -42,26 +43,27 @@ func WithOffline(offline bool) option {
 	}
 }
 
-func WithRemoteRepos(repos []string) option {
+func WithReleaseRemoteRepos(repos []string) option {
 	return func(opts *options) {
-		opts.remoteRepos = repos
+		opts.releaseRemoteRepos = repos
 	}
 }
 
 type parser struct {
-	logger             *log.Logger
-	rootPath           string
-	cache              pomCache
-	localRepository    string
-	remoteRepositories []string
-	offline            bool
-	servers            []Server
+	logger              *log.Logger
+	rootPath            string
+	cache               pomCache
+	localRepository     string
+	releaseRemoteRepos  []string
+	snapshotRemoteRepos []string
+	offline             bool
+	servers             []Server
 }
 
 func NewParser(filePath string, opts ...option) types.Parser {
 	o := &options{
-		offline:     false,
-		remoteRepos: []string{centralURL},
+		offline:            false,
+		releaseRemoteRepos: []string{centralURL}, // Maven doesn't use central repository for snapshot dependencies
 	}
 
 	for _, opt := range opts {
@@ -76,13 +78,14 @@ func NewParser(filePath string, opts ...option) types.Parser {
 	}
 
 	return &parser{
-		logger:             log.WithPrefix("pom"),
-		rootPath:           filepath.Clean(filePath),
-		cache:              newPOMCache(),
-		localRepository:    localRepository,
-		remoteRepositories: o.remoteRepos,
-		offline:            o.offline,
-		servers:            s.Servers,
+		logger:              log.WithPrefix("pom"),
+		rootPath:            filepath.Clean(filePath),
+		cache:               newPOMCache(),
+		localRepository:     localRepository,
+		releaseRemoteRepos:  o.releaseRemoteRepos,
+		snapshotRemoteRepos: o.snapshotRemoteRepos,
+		offline:             o.offline,
+		servers:             s.Servers,
 	}
 }
 
@@ -324,7 +327,9 @@ func (p *parser) analyze(pom *pom, opts analysisOptions) (analysisResult, error)
 	}
 
 	// Update remoteRepositories
-	p.remoteRepositories = utils.UniqueStrings(append(pom.repositories(p.servers), p.remoteRepositories...))
+	pomReleaseRemoteRepos, pomSnapshotRemoteRepos := pom.repositories(p.servers)
+	p.releaseRemoteRepos = lo.Uniq(append(pomReleaseRemoteRepos, p.releaseRemoteRepos...))
+	p.snapshotRemoteRepos = lo.Uniq(append(pomSnapshotRemoteRepos, p.snapshotRemoteRepos...))
 
 	// Parent
 	parent, err := p.parseParent(pom.filePath, pom.content.Parent)
@@ -615,7 +620,7 @@ func (p *parser) tryRepository(groupID, artifactID, version string) (*pom, error
 	}
 
 	// Search remote remoteRepositories
-	loaded, err = p.fetchPOMFromRemoteRepositories(paths)
+	loaded, err = p.fetchPOMFromRemoteRepositories(paths, isSnapshot(version))
 	if err == nil {
 		return loaded, nil
 	}
@@ -630,15 +635,21 @@ func (p *parser) loadPOMFromLocalRepository(paths []string) (*pom, error) {
 	return p.openPom(localPath)
 }
 
-func (p *parser) fetchPOMFromRemoteRepositories(paths []string) (*pom, error) {
+func (p *parser) fetchPOMFromRemoteRepositories(paths []string, snapshot bool) (*pom, error) {
 	// Do not try fetching pom.xml from remote repositories in offline mode
 	if p.offline {
 		p.logger.Debug("Fetching the remote pom.xml is skipped")
 		return nil, xerrors.New("offline mode")
 	}
 
+	remoteRepos := p.releaseRemoteRepos
+	// Maven uses only snapshot repos for snapshot artifacts
+	if snapshot {
+		remoteRepos = p.snapshotRemoteRepos
+	}
+
 	// try all remoteRepositories
-	for _, repo := range p.remoteRepositories {
+	for _, repo := range remoteRepos {
 		fetched, err := p.fetchPOMFromRemoteRepository(repo, paths)
 		if err != nil {
 			return nil, xerrors.Errorf("fetch repository error: %w", err)
@@ -703,3 +714,8 @@ func parsePom(r io.Reader) (*pomXML, error) {
 func packageID(name, version string) string {
 	return dependency.ID(ftypes.Pom, name, version)
 }
+
+// cf. https://github.com/apache/maven/blob/259404701402230299fe05ee889ecdf1c9dae816/maven-artifact/src/main/java/org/apache/maven/artifact/DefaultArtifact.java#L482-L486
+func isSnapshot(ver string) bool {
+	return strings.HasSuffix(ver, "SNAPSHOT") || ver == "LATEST"
+}

pkg/dependency/parser/java/pom/parse_test.go

@@ -70,7 +70,7 @@ func TestPom_Parse(t *testing.T) {
 			},
 		},
 		{
-			name:      "remote repository",
+			name:      "remote release repository",
 			inputFile: filepath.Join("testdata", "happy", "pom.xml"),
 			local:     false,
 			want: []types.Library{
@@ -114,6 +114,37 @@ func TestPom_Parse(t *testing.T) {
 				},
 			},
 		},
+		{
+			name:      "snapshot dependency",
+			inputFile: filepath.Join("testdata", "snapshot", "pom.xml"),
+			local:     false,
+			want: []types.Library{
+				{
+					ID:      "com.example:happy:1.0.0",
+					Name:    "com.example:happy",
+					Version: "1.0.0",
+				},
+				{
+					ID:      "org.example:example-dependency:1.2.3-SNAPSHOT",
+					Name:    "org.example:example-dependency",
+					Version: "1.2.3-SNAPSHOT",
+					Locations: types.Locations{
+						{
+							StartLine: 14,
+							EndLine:   18,
+						},
+					},
+				},
+			},
+			wantDeps: []types.Dependency{
+				{
+					ID: "com.example:happy:1.0.0",
+					DependsOn: []string{
+						"org.example:example-dependency:1.2.3-SNAPSHOT",
+					},
+				},
+			},
+		},
 		{
 			name:      "offline mode",
 			inputFile: filepath.Join("testdata", "offline", "pom.xml"),
@@ -1295,7 +1326,7 @@ func TestPom_Parse(t *testing.T) {
 				remoteRepos = []string{ts.URL}
 			}
 
-			p := pom.NewParser(tt.inputFile, pom.WithRemoteRepos(remoteRepos), pom.WithOffline(tt.offline))
+			p := pom.NewParser(tt.inputFile, pom.WithReleaseRemoteRepos(remoteRepos), pom.WithOffline(tt.offline))
 
 			gotLibs, gotDeps, err := p.Parse(f)
 			if tt.wantErr != "" {

pkg/dependency/parser/java/pom/pom.go

@@ -115,12 +115,14 @@ func (p pom) licenses() []string {
 	})
 }
 
-func (p pom) repositories(servers []Server) []string {
+func (p pom) repositories(servers []Server) ([]string, []string) {
 	logger := log.WithPrefix("pom")
-	var urls []string
+	var releaseRepos, snapshotRepos []string
 	for _, rep := range p.content.Repositories.Repository {
+		snapshot := rep.Snapshots.Enabled == "true"
+		release := rep.Releases.Enabled == "true"
 		// Add only enabled repositories
-		if rep.Releases.Enabled == "false" && rep.Snapshots.Enabled == "false" {
+		if !release && !snapshot {
 			continue
 		}
 
@@ -140,9 +142,15 @@ func (p pom) repositories(servers []Server) []string {
 		}
 
 		logger.Debug("Adding repository", log.String("id", rep.ID), log.String("url", rep.URL))
-		urls = append(urls, repoURL.String())
+		if snapshot {
+			snapshotRepos = append(snapshotRepos, repoURL.String())
+		}
+		if release {
+			releaseRepos = append(releaseRepos, repoURL.String())
+		}
 	}
-	return urls
+
+	return releaseRepos, snapshotRepos
 }
 
 type pomXML struct {

pkg/dependency/parser/java/pom/testdata/repository/org/example/example-dependency/1.2.3-SNAPSHOT/example-dependency-1.2.3.pom

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>org.example</groupId>
+    <artifactId>example-dependency</artifactId>
+    <version>1.2.3-SNAPSHOT</version>
+
+    <packaging>jar</packaging>
+    <name>Example API Dependency</name>
+    <description>The example API</description>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.example</groupId>
+            <artifactId>example-api</artifactId>
+            <version>2.0.0</version>
+        </dependency>
+    </dependencies>
+
+</project>

pkg/dependency/parser/java/pom/testdata/snapshot/pom.xml

@@ -0,0 +1,20 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>com.example</groupId>
+    <artifactId>happy</artifactId>
+    <version>1.0.0</version>
+
+    <name>happy</name>
+    <description>Example</description>
+
+
+    <dependencies>
+        <dependency>
+            <groupId>org.example</groupId>
+            <artifactId>example-dependency</artifactId>
+            <version>1.2.3-SNAPSHOT</version>
+        </dependency>
+    </dependencies>
+</project>