feat: add download rulesets handler

.gitignore

@@ -43,3 +43,5 @@ local-build-commands.txt
 *.log
 
 .DS_Store
+
+!**/rulesets/**/*.tar.gz

CHANGELOG.md

@@ -4,6 +4,17 @@
 ### Added
 - Upcoming changes...
 
+## [0.9.0] - 2025-12-29
+### Added
+- Added gRPC DownloadRuleset and REST endpoint GET /v2/cryptography/rulesets/download
+  - Supports downloading cryptography detection rulesets by name and version
+  - Supports "latest" keyword for retrieving the most recent version
+  - Returns tarball with appropriate HTTP headers (Content-Disposition, SCANOSS-Ruleset-Name, SCANOSS-Ruleset-Version, X-Checksum-SHA256)
+  - Includes metadata validation and version resolution via symlinks
+
+### Fixed
+- Fixed OpenTelemetry metrics initialization by properly exporting SetupMetrics function 
+
 ## [0.8.1] - 2025-10-16
 ### Fixed
 - Fixed OpenTelemetry metrics initialization by exporting SetupMetrics function and calling it on server startup
@@ -96,6 +107,7 @@
 - Remove from list those versions that do not contain detections
 - Detailed response status message.
 
+[0.9.0]: https://github.com/scanoss/cryptography/compare/v0.8.1...v0.9.0
 [0.8.1]: https://github.com/scanoss/cryptography/compare/v0.8.0...v0.8.1
 [0.8.0]: https://github.com/scanoss/cryptography/compare/v0.7.1...v0.8.0
 [0.7.1]: https://github.com/scanoss/cryptography/compare/v0.7.0...v0.7.1
@@ -104,4 +116,4 @@
 [0.5.0]: https://github.com/scanoss/cryptography/compare/v0.4.2...v0.5.0
 [0.4.2]: https://github.com/scanoss/cryptography/releases/tag/v0.4.1....v0.4.2
 [0.4.1]: https://github.com/scanoss/cryptography/releases/tag/v0.4.0...v0.4.1
-[0.4.0]: https://github.com/scanoss/cryptography/releases/tag/v0.4.0
+[0.4.0]: https://github.com/scanoss/cryptography/releases/tag/v0.4.0

config/app-config-dev.json

@@ -2,11 +2,14 @@
   "App": {
     "Name": "Cryptography Service",
     "Debug": true,
-    "Mode":  "dev"
+    "Mode": "dev"
   },
   "Database": {
     "Dsn": "./test-support/sqlite/scanoss.db?cache=shared&mode=memory",
     "Driver": "sqlite",
     "Trace": true
+  },
+  "Rulesets": {
+    "StoragePath": "test-support/rulesets"
   }
 }

go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/lib/pq v1.10.9
 	github.com/scanoss/go-grpc-helper v0.9.0
 	github.com/scanoss/go-purl-helper v0.2.1
-	github.com/scanoss/papi v0.25.1
+	github.com/scanoss/papi v0.27.0
 	github.com/scanoss/zap-logging-helper v0.4.0
 	github.com/stretchr/testify v1.11.1
 	go.opentelemetry.io/otel v1.38.0
@@ -37,7 +37,7 @@ require (
 	go.opentelemetry.io/otel/sdk/metric v1.37.0 // indirect
 	go.opentelemetry.io/otel/trace v1.38.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.6.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20250707201910-8d1bb00bc6a7 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20250707201910-8d1bb00bc6a7
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect
 )
 
@@ -70,6 +70,6 @@ require (
 // Details of how to use the "replace" command for local development
 // https://github.com/golang/go/wiki/Modules#when-should-i-use-the-replace-directive
 // ie. replace github.com/scanoss/papi => ../papi
-// replace github.com/scanoss/papi => ../papi
+//replace github.com/scanoss/papi => ../papi
 
 // require github.com/scanoss/papi v0.0.0-unpublished

go.sum

@@ -626,8 +626,8 @@ github.com/scanoss/go-purl-helper v0.2.1 h1:jp960a585ycyJSlqZky1NatMJBIQi/JGITDf
 github.com/scanoss/go-purl-helper v0.2.1/go.mod h1:v20/bKD8G+vGrILdiq6r0hyRD2bO8frCJlu9drEcQ38=
 github.com/scanoss/ipfilter/v2 v2.0.2 h1:GaB9i8kVJg9JQZm5XGStYkEpiaCVdsrj7ezI2wV/oh8=
 github.com/scanoss/ipfilter/v2 v2.0.2/go.mod h1:AwrpX4XGbZ7EKISMi1d6E5csBk1nWB8+ugpvXHFcTpA=
-github.com/scanoss/papi v0.25.1 h1:/OUoCWkrD+PRNvssSfrVcgPFFHKl3rCp/zjz0I2qNd8=
-github.com/scanoss/papi v0.25.1/go.mod h1:Z4E/4IpwYdzHHRJXTgBCGG1GjksgrFjNW5cvhbKUfeU=
+github.com/scanoss/papi v0.27.0 h1:raPAm9aFmcGVHdYZh+7su20R0ow+9tpI3jMDxxQaFUM=
+github.com/scanoss/papi v0.27.0/go.mod h1:Z4E/4IpwYdzHHRJXTgBCGG1GjksgrFjNW5cvhbKUfeU=
 github.com/scanoss/zap-logging-helper v0.4.0 h1:2qTYoaFa9+MlD2/1wmPtiDHfh+42NIEwgKVU3rPpl0Y=
 github.com/scanoss/zap-logging-helper v0.4.0/go.mod h1:9QuEZcq73g/0Izv1tWeOWukoIK0oTBzM4jSNQ5kRR1w=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=

pkg/config/server_config.go

@@ -17,6 +17,9 @@
 package config
 
 import (
+	"fmt"
+	"os"
+
 	"github.com/golobby/config/v3"
 	"github.com/golobby/config/v3/pkg/feeder"
 )
@@ -68,6 +71,9 @@ type ServerConfig struct {
 		BlockByDefault bool   `env:"CRYPTO_BLOCK_BY_DEFAULT"` // Block request by default if they are not in the allow list
 		TrustProxy     bool   `env:"CRYPTO_TRUST_PROXY"`      // Trust the interim proxy or not (causes the source IP to be validated instead of the proxy)
 	}
+	Rulesets struct {
+		StoragePath string `env:"RULESETS_STORAGE_PATH"` // Path to directory containing cryptography detection rulesets
+	}
 }
 
 // NewServerConfig loads all config options and return a struct for use.
@@ -84,6 +90,9 @@ func NewServerConfig(feeders []config.Feeder) (*ServerConfig, error) {
 	if err != nil {
 		return nil, err
 	}
+	if err := cfg.ValidateRulesetsFolder(); err != nil {
+		return nil, err
+	}
 	return &cfg, nil
 }
 
@@ -105,4 +114,26 @@ func setServerConfigDefaults(cfg *ServerConfig) {
 	cfg.Logging.DynamicPort = "localhost:60054"
 	cfg.Telemetry.Enabled = false
 	cfg.Telemetry.OltpExporter = "0.0.0.0:4317" // Default OTEL OLTP gRPC Exporter endpoint
+	cfg.Rulesets.StoragePath = "/var/lib/scanoss/cryptography/rulesets"
+}
+
+// ValidateRulesetsFolder validates that the configured rulesets storage path exists and is a directory.
+func (cfg *ServerConfig) ValidateRulesetsFolder() error {
+	if cfg.Rulesets.StoragePath == "" {
+		return fmt.Errorf("rulesets storage path is not configured")
+	}
+
+	info, err := os.Stat(cfg.Rulesets.StoragePath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return fmt.Errorf("rulesets storage path does not exist: %s", cfg.Rulesets.StoragePath)
+		}
+		return fmt.Errorf("failed to access rulesets storage path: %w", err)
+	}
+
+	if !info.IsDir() {
+		return fmt.Errorf("rulesets storage path is not a directory: %s", cfg.Rulesets.StoragePath)
+	}
+
+	return nil
 }

pkg/config/server_config_test.go

@@ -23,14 +23,19 @@ import (
 
 	"github.com/golobby/config/v3"
 	"github.com/golobby/config/v3/pkg/feeder"
+	"scanoss.com/cryptography/pkg/testutils"
 )
 
 func TestServerConfig(t *testing.T) {
+	defer testutils.SetupTestRulesetsDir(t)()
+
 	dbUser := "test-user"
 	err := os.Setenv("DB_USER", dbUser)
 	if err != nil {
 		t.Fatalf("an error '%s' was not expected when creating new config instance", err)
 	}
+	defer os.Unsetenv("DB_USER")
+
 	cfg, err := NewServerConfig(nil)
 	if err != nil {
 		t.Fatalf("an error '%s' was not expected when creating new config instance", err)
@@ -39,17 +44,16 @@ func TestServerConfig(t *testing.T) {
 		t.Errorf("DB user '%v' doesn't match expected: %v", cfg.Database.User, dbUser)
 	}
 	fmt.Printf("Server Config1: %+v\n", cfg)
-	err = os.Unsetenv("DB_USER")
-	if err != nil {
-		fmt.Printf("Warning: Problem runn Unsetenv: %v\n", err)
-	}
 }
 
 func TestServerConfigDotEnv(t *testing.T) {
+	defer testutils.SetupTestRulesetsDir(t)()
+
 	err := os.Unsetenv("DB_USER")
 	if err != nil {
 		fmt.Printf("Warning: Problem runn Unsetenv: %v\n", err)
 	}
+
 	dbUser := "env-user"
 	var feeders []config.Feeder
 	feeders = append(feeders, feeder.DotEnv{Path: "tests/dot-env"})
@@ -64,10 +68,13 @@ func TestServerConfigDotEnv(t *testing.T) {
 }
 
 func TestServerConfigJson(t *testing.T) {
+	defer testutils.SetupTestRulesetsDir(t)()
+
 	err := os.Unsetenv("DB_USER")
 	if err != nil {
 		fmt.Printf("Warning: Problem runn Unsetenv: %v\n", err)
 	}
+
 	dbUser := "json-user"
 	var feeders []config.Feeder
 	feeders = append(feeders, feeder.Json{Path: "tests/env.json"})
@@ -80,3 +87,70 @@ func TestServerConfigJson(t *testing.T) {
 	}
 	fmt.Printf("Server Config3: %+v\n", cfg)
 }
+
+func TestValidateRulesetsFolder(t *testing.T) {
+	// Test with non-existent path
+	t.Run("NonExistentPath", func(t *testing.T) {
+		err := os.Setenv("RULESETS_STORAGE_PATH", "/path/that/does/not/exist")
+		if err != nil {
+			t.Fatalf("failed to set RULESETS_STORAGE_PATH: %s", err)
+		}
+		defer os.Unsetenv("RULESETS_STORAGE_PATH")
+
+		_, err = NewServerConfig(nil)
+		if err == nil {
+			t.Fatal("expected error for non-existent rulesets path, got nil")
+		}
+		if !os.IsNotExist(err) && err.Error() != "rulesets storage path does not exist: /path/that/does/not/exist" {
+			t.Errorf("unexpected error message: %v", err)
+		}
+	})
+
+	// Test with valid directory
+	t.Run("ValidDirectory", func(t *testing.T) {
+		tmpDir, err := os.MkdirTemp("", "rulesets-test-*")
+		if err != nil {
+			t.Fatalf("failed to create temp directory: %s", err)
+		}
+		defer os.RemoveAll(tmpDir)
+
+		err = os.Setenv("RULESETS_STORAGE_PATH", tmpDir)
+		if err != nil {
+			t.Fatalf("failed to set RULESETS_STORAGE_PATH: %s", err)
+		}
+		defer os.Unsetenv("RULESETS_STORAGE_PATH")
+
+		cfg, err := NewServerConfig(nil)
+		if err != nil {
+			t.Fatalf("unexpected error for valid rulesets path: %s", err)
+		}
+		if cfg.Rulesets.StoragePath != tmpDir {
+			t.Errorf("expected storage path %s, got %s", tmpDir, cfg.Rulesets.StoragePath)
+		}
+	})
+
+	// Test with file instead of directory
+	t.Run("FileInsteadOfDirectory", func(t *testing.T) {
+		tmpFile, err := os.CreateTemp("", "rulesets-file-*")
+		if err != nil {
+			t.Fatalf("failed to create temp file: %s", err)
+		}
+		tmpFile.Close()
+		defer os.Remove(tmpFile.Name())
+
+		err = os.Setenv("RULESETS_STORAGE_PATH", tmpFile.Name())
+		if err != nil {
+			t.Fatalf("failed to set RULESETS_STORAGE_PATH: %s", err)
+		}
+		defer os.Unsetenv("RULESETS_STORAGE_PATH")
+
+		_, err = NewServerConfig(nil)
+		if err == nil {
+			t.Fatal("expected error for file instead of directory, got nil")
+		}
+		expectedMsg := fmt.Sprintf("rulesets storage path is not a directory: %s", tmpFile.Name())
+		if err.Error() != expectedMsg {
+			t.Errorf("expected error message '%s', got '%s'", expectedMsg, err.Error())
+		}
+	})
+}

pkg/handlers/algorithm_handler_test.go

@@ -27,9 +27,12 @@ import (
 	_ "modernc.org/sqlite"
 	myconfig "scanoss.com/cryptography/pkg/config"
 	"scanoss.com/cryptography/pkg/models"
+	"scanoss.com/cryptography/pkg/testutils"
 )
 
 func TestNewCryptographyAlgorithmHandler(t *testing.T) {
+	defer testutils.SetupTestRulesetsDir(t)()
+
 	err := zlog.NewSugaredDevLogger()
 	if err != nil {
 		t.Fatalf("failed to initialize logger: %v", err)
@@ -57,6 +60,8 @@ func TestNewCryptographyAlgorithmHandler(t *testing.T) {
 }
 
 func TestCryptographyAlgorithmHandler_GetAlgorithms(t *testing.T) {
+	defer testutils.SetupTestRulesetsDir(t)()
+
 	ctx := context.Background()
 	err := zlog.NewSugaredDevLogger()
 	if err != nil {
@@ -176,6 +181,8 @@ func TestCryptographyAlgorithmHandler_GetAlgorithms(t *testing.T) {
 }
 
 func TestCryptographyAlgorithmHandler_GetComponentsAlgorithms(t *testing.T) {
+	defer testutils.SetupTestRulesetsDir(t)()
+
 	ctx := context.Background()
 	err := zlog.NewSugaredDevLogger()
 	if err != nil {
@@ -313,6 +320,8 @@ func TestCryptographyAlgorithmHandler_GetComponentsAlgorithms(t *testing.T) {
 }
 
 func TestCryptographyAlgorithmHandler_GetComponentAlgorithms(t *testing.T) {
+	defer testutils.SetupTestRulesetsDir(t)()
+
 	ctx := context.Background()
 	err := zlog.NewSugaredDevLogger()
 	if err != nil {

pkg/handlers/algorithm_in_range_handler_test.go

@@ -27,9 +27,12 @@ import (
 	_ "modernc.org/sqlite"
 	myconfig "scanoss.com/cryptography/pkg/config"
 	"scanoss.com/cryptography/pkg/models"
+	"scanoss.com/cryptography/pkg/testutils"
 )
 
 func TestNewAlgorithmInRangeHandler(t *testing.T) {
+	defer testutils.SetupTestRulesetsDir(t)()
+
 	err := zlog.NewSugaredDevLogger()
 	if err != nil {
 		t.Fatalf("failed to initialize logger: %v", err)
@@ -57,6 +60,8 @@ func TestNewAlgorithmInRangeHandler(t *testing.T) {
 }
 
 func TestAlgorithmInRangeHandler_GetAlgorithmsInRange(t *testing.T) {
+	defer testutils.SetupTestRulesetsDir(t)()
+
 	ctx := context.Background()
 	err := zlog.NewSugaredDevLogger()
 	if err != nil {
@@ -176,6 +181,8 @@ func TestAlgorithmInRangeHandler_GetAlgorithmsInRange(t *testing.T) {
 }
 
 func TestAlgorithmInRangeHandler_GetComponentsAlgorithmsInRange(t *testing.T) {
+	defer testutils.SetupTestRulesetsDir(t)()
+
 	ctx := context.Background()
 	err := zlog.NewSugaredDevLogger()
 	if err != nil {
@@ -313,6 +320,8 @@ func TestAlgorithmInRangeHandler_GetComponentsAlgorithmsInRange(t *testing.T) {
 }
 
 func TestAlgorithmInRangeHandler_GetComponentAlgorithmsInRange(t *testing.T) {
+	defer testutils.SetupTestRulesetsDir(t)()
+
 	ctx := context.Background()
 	err := zlog.NewSugaredDevLogger()
 	if err != nil {

pkg/handlers/cryptography_support.go

@@ -35,18 +35,21 @@ import (
 // about handler performance and request processing times.
 type metricsCounters struct {
 	cryptoAlgorithmsHistogram metric.Int64Histogram // Histogram for recording crypto algorithms request times in milliseconds
+	downloadRulesetHistogram  metric.Int64Histogram // Histogram for recording ruleset download request times in milliseconds
+	downloadRulesetCounter    metric.Int64Counter   // Counter for tracking the number of downloaded rulesets
 }
 
 var oltpMetrics = metricsCounters{}
 
-// setupMetrics configures all OpenTelemetry metric instruments for the handlers package.
+// SetupMetrics configures all OpenTelemetry metric instruments for the handlers package.
 //
 // This function initializes histogram metrics for tracking request durations.
 // It should be called once during handler initialization to set up the metrics infrastructure.
-
 func SetupMetrics() {
 	meter := otel.Meter("scanoss.com/cryptography")
 	oltpMetrics.cryptoAlgorithmsHistogram, _ = meter.Int64Histogram("crypto.algorithms.req_time", metric.WithDescription("The time taken to run a crypto algorithms request (ms)"))
+	oltpMetrics.downloadRulesetHistogram, _ = meter.Int64Histogram("crypto.rulesets.download_time", metric.WithDescription("The time taken to download a ruleset (ms)"))
+	oltpMetrics.downloadRulesetCounter, _ = meter.Int64Counter("crypto.rulesets.downloaded", metric.WithDescription("The number of downloaded rulesets"))
 }
 
 // ConvertPurlRequestToComponentDTO converts a legacy PurlRequest to ComponentDTO slice.