5.4.10 (#85)

* fix memory bug processing versions

* change sources env-variables

* tune up scanning limits

* fix ranges assembling bug. Improve snippets and component selection

* change on report, file content url will be returned empty if the env-var wasn't defined

Author: mscasso-scanoss

inc/limits.h

@@ -37,9 +37,9 @@
 
 /* Snippets */
 #define DEFAULT_MATCHMAP_FILES 10000     // Default number of files evaluated in snippet matching
-#define MAX_MATCHMAP_FILES (DEFAULT_MATCHMAP_FILES * 5)     // Max number of files evaluated in snippet matching to prevent performance issues
+#define MAX_MATCHMAP_FILES (DEFAULT_MATCHMAP_FILES * 10)     // Max number of files evaluated in snippet matching to prevent performance issues
+#define MIN_LINES_COVERAGE 0.8
 #define SKIP_SNIPPETS_IF_FILE_BIGGER (1024 * 1024 * 4)
-#define SKIP_SNIPPETS_IF_STARTS_WITH (const char*[3]) {"{", "<?xml", "<html"}
 #define MAX_SNIPPETS_SCANNED 2500
 
 /* Variables */

inc/match.h

@@ -13,6 +13,7 @@ typedef struct match_data_t
 	component_list_t component_list; /*Component list object */ 
 	match_t type; /*math type (none, snippet, file) */
     int hits; /*match hits number, more hits equal bigger snippet matching*/
+	int lines_matched; /*number of matched lines*/
 	char * line_ranges; /*input snippet line ranges */
 	char * oss_ranges; /* kb snippet line ranges */
 	char * matched_percent; /* matched percent */

inc/match_list.h

@@ -78,7 +78,7 @@
 #define SCAN_MAX_SNIPPETS_DEFAULT 	1
 #define SCAN_MAX_COMPONENTS_DEFAULT 3
 
-#define MATCH_LIST_TOLERANCE 97.5
+#define MATCH_LIST_TOLERANCE 99.9
 typedef struct match_data_t match_data_t; /* Forward declaration */
 
 /**
@@ -145,6 +145,7 @@ bool component_list_add(component_list_t * list, component_data_t * new_comp, bo
 void component_list_print(component_list_t * list, bool (*printer) (component_data_t * fpa), char * separator);
 void component_list_destroy(component_list_t *list);
 bool component_list_add_binary(component_list_t *list, component_data_t *new_comp, bool (*val)(component_data_t *a, component_data_t *b), bool remove_a);
+bool match_list_eval(match_list_t *list, match_data_t * in,  bool (*eval)(match_data_t *fpa, match_data_t *fpb));
 void match_list_tolerance_set(float in);
 
 #endif

inc/scan.h

@@ -52,7 +52,7 @@ typedef struct scan_data_t
 	match_t match_type; /* match_t (file, snippet, none), this is replicated in each match in the matches list */
 	matchmap_entry *matchmap; /*matchmap pointer, used in snippet scanning */
 	uint32_t matchmap_size; /*size of the match map */
-	int matchmap_rank_by_sector[255]; /* Indirection array pointing to the max hits from the matchmap classyfied by sector.*/
+	int matchmap_rank_by_sector[256]; /* Indirection array pointing to the max hits from the matchmap classyfied by sector.*/
 	uint8_t *match_ptr; // pointer to matching record in match_map
 	match_list_t * matches_list_array[MAX_MULTIPLE_COMPONENTS]; /* array of "match_list_t", each snippet with different "from line" will generate its own matches list */
 	int matches_list_array_index; /* elements in the matches list array*/

inc/scanoss.h

@@ -40,12 +40,11 @@
 #define WFP_REC_LN 18
 
 /* Log files */
-#define SCANOSS_VERSION "5.4.9"
+#define SCANOSS_VERSION "5.4.10"
 #define SCAN_LOG "/tmp/scanoss_scan.log"
 #define MAP_DUMP "/tmp/scanoss_map.dump"
 #define SLOW_QUERY_LOG "/tmp/scanoss_slow_query.log"
 
-#define API_URL "https://api.osskb.org"
 #define DEFAULT_OSS_DB_NAME "oss"
 
 /* Engine configuration flags */

src/help.c

@@ -50,7 +50,7 @@ Configuration:\n\
 -H         Enable High Precision Snippet Match mode (requires 'libhpsm.so' in the system).\n\
 -e         Match only files with identical extensions as the scanned file (default: off).\n\
 -M NUMBER  Search for up to NUMBER different components in each file (maximum: 9).\n\
--T NUMBER  Set snippet scanning tolerance percentage (default: 3.5).\n\
+-T NUMBER  Set snippet scanning tolerance percentage (default: 0.1).\n\
 -s SBOM    Include assets from a JSON SBOM file (CycloneDX/SPDX2.2 format) in identification.\n\
 -b SBOM    Exclude matches from assets listed in JSON SBOM file (CycloneDX/SPDX2.2 format).\n\
 -B SBOM    Same as \"-b\" but with forced snippet scanning.\n\
@@ -69,7 +69,7 @@ Options:\n\
 \n\
 Environment variables:\n\
 SCANOSS_MATCHMAP_MAX: Set the snippet scanning match map size (default: %d).\n\
-SCANOSS_API_URL: Define the API endpoint URL (default: %s).\n\
+SCANOSS_FILE_CONTENTS_URL: Define the API URL endpoint for sources. Source url wont be reported if it's not defined.\n\
 \n\
 Engine scanning flags:\n\
 Configure the scanning engine using flags with the -F parameter.\n\
@@ -95,5 +95,5 @@ These settings can also be specified in %s\n\
 +-------+-------------------------------------------------------+\n\
 Example: scanoss -F 12 DIRECTORY (scan DIRECTORY without license and dependency data)\n\
 \n\
-Copyright (C) 2018-2022 SCANOSS.COM\n", DEFAULT_MATCHMAP_FILES, API_URL, ENGINE_FLAGS_FILE);
+Copyright (C) 2018-2022 SCANOSS.COM\n", DEFAULT_MATCHMAP_FILES, ENGINE_FLAGS_FILE);
 }

src/match.c

@@ -320,7 +320,7 @@ static bool component_hint_date_comparation(component_data_t *a, component_data_
 	if (!*a->release_date)
 		return true;
 
-	if (!path_is_third_party(a->file) && path_is_third_party(b->file) && !(engine_flags & ENABLE_PATH_HINT))
+	if (!path_is_third_party(a->file) && path_is_third_party(b->file))
 	{
 		scanlog("Component rejected by third party filter\n");
 		return false;
@@ -331,7 +331,7 @@ static bool component_hint_date_comparation(component_data_t *a, component_data_
 	{	
 		if (purl_source_check(a) > purl_source_check(b))
 		{
-			scanlog("Component prefered by vsource\n");
+			scanlog("Component prefered by source\n");
 			return true;
 		}
 
@@ -471,39 +471,15 @@ bool load_matches(match_data_t *match)
 {
 	scanlog("Load matches\n");
 
-	/* Compile match ranges and fill up matched percent */
-	int hits = 100;
-	int matched_percent = 100;
 
-	/* Get matching line ranges (snippet match) */
-	if (match->type == MATCH_SNIPPET)
-	{
-		hits = compile_ranges(match);
-		scanlog("compile_ranges returns %d hits\n", hits);
-
-		if (hits < min_match_hits)
-		{
-			match->type = MATCH_NONE;
-			return false;
-		}
-		
-		float percent = (hits * 100) / match->scan_ower->total_lines;
-		if (hits)
-			matched_percent = floor(percent);
-		if (matched_percent > 99)
-			matched_percent = 99;
-		if (matched_percent < 1)
-			matched_percent = 1;
-
-		asprintf(&match->matched_percent, "%u%%", matched_percent);
-	}
-	else if (match->type == MATCH_BINARY)
+	
+	if (match->type == MATCH_BINARY)
 	{
 		asprintf(&match->line_ranges, "n/a");
 		asprintf(&match->oss_ranges, "n/a");
 		asprintf(&match->matched_percent, "%d functions matched", match->hits);
 	}
-	else
+	else if (match->type == MATCH_FILE)
 	{
 		asprintf(&match->line_ranges, "all");
 		asprintf(&match->oss_ranges, "all");
@@ -696,11 +672,13 @@ void match_select_best(scan_data_t *scan)
 				break;
 			}
 
-			if (!best_match_component->identified && match_component->identified)
+			if ((!best_match_component->identified && match_component->identified) ||
+				(strcmp(best_match_component->vendor,best_match_component->component) && !strcmp(match_component->vendor, match_component->component)) ||
+				(path_is_third_party(best_match_component->file) && !path_is_third_party(match_component->file)))
 			{
 				scanlog("Replacing best match for a prefered component\n");
 				scan->matches_list_array[i]->best_match = item->match;
-			}	
+			}
 		}
 	}
 

src/match_list.c

@@ -311,11 +311,11 @@ bool match_list_add(match_list_t *list, match_data_t *new_match, bool (*val)(mat
         }
         /* in autolimit mode the list doesnt have a fix size, it will accept all the matchest until a 75% of the fist element (the biggest) */
         //TODO: this part of the code should be in the function pointer or I need to re-evaluate the archtecture of this function */
-        if (list->autolimit && !tolerance_eval(list->headp.lh_first->match->hits, list->last_element->match->hits))
+        if (list->autolimit && !tolerance_eval(list->headp.lh_first->match->lines_matched, list->last_element->match->lines_matched))
         {    
             np = list->headp.lh_first;
             /*We have to find and remove the unwanted elements */
-            for (; np->entries.le_next != NULL && tolerance_eval(list->headp.lh_first->match->hits, np->entries.le_next->match->hits); np = np->entries.le_next)
+            for (; np->entries.le_next != NULL && tolerance_eval(list->headp.lh_first->match->lines_matched, np->entries.le_next->match->lines_matched); np = np->entries.le_next)
             {
 
             }
@@ -403,6 +403,18 @@ bool match_list_print(match_list_t *list, bool (*printer)(match_data_t *fpa), ch
     return true;
 }
 
+bool match_list_eval(match_list_t *list, match_data_t * in,  bool (*eval)(match_data_t *fpa, match_data_t *fpb))
+{
+    int i = 0;
+    for (struct entry *np = list->headp.lh_first; np != NULL && i<list->items; np = np->entries.le_next)
+    {
+        if(eval(np->match, in))
+            return true;
+        i++;
+    }
+    return false;
+}
+
 void component_list_print(component_list_t *list, bool (*printer)(component_data_t *fpa), char *separator)
 {
     for (struct comp_entry *np = list->headp.lh_first; np != NULL; np = np->entries.le_next)

src/report.c

@@ -334,17 +334,19 @@ bool print_json_match(struct match_data_t * match)
 	printf(",\"source_hash\": \"%s\"", match->source_md5);
 
 	/* Output file_url (same as url when match type = url) */
-	char * file_url_enabled = getenv("SCANOSS_FILE_CONTENTS");
-	if (!file_url_enabled || strcmp(file_url_enabled, "false"))
+	char * file_contents_url = getenv("SCANOSS_FILE_CONTENTS_URL");
+	if (file_contents_url && *file_contents_url && strcmp(file_contents_url, "false"))
 	{
 		if (!match->component_list.headp.lh_first->component->url_match)
 		{
-			char *custom_url = getenv("SCANOSS_API_URL");
-			printf(",\"file_url\": \"%s/file_contents/%s\"", custom_url ? custom_url : API_URL, file_id);
+			printf(",\"file_url\": \"%s/%s\"", file_contents_url, file_id);
 		}
 		else
 			printf(",\"file_url\": \"%s\"", match->component_list.headp.lh_first->component->url);
 	}
+	else //return an empty string
+		printf(",\"file_url\": \" \"");
+
 
 	free(file_id);
 

src/scan.c

@@ -59,7 +59,7 @@ scan_data_t * scan_data_init(char *target, int max_snippets, int max_components)
 	scan_data_t * scan = calloc(1, sizeof(*scan));
 	scan->file_path = strdup(target);
 	scan->file_size = malloc(32);
-	scan->hashes = malloc(MAX_FILE_SIZE);
+	scan->hashes = calloc(MAX_FILE_SIZE,1);
 	scan->lines  = malloc(MAX_FILE_SIZE);
 	scan->match_type = MATCH_NONE;