Skip to content

[CYB-183] Marked output fields where needed #60

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
stas-panasiuk merged 5 commits into from
Apr 21, 2024
Merged

[CYB-183] Marked output fields where needed #60

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
cc7700b
[CYB-183] Marked output fields where needed
stas-panasiuk Oct 19, 2023
5ca8ad3
Merge branch 'develop' into CYB-183
stas-panasiuk Feb 16, 2024
0666125
Merge branch 'develop' into CYB-183
stas-panasiuk Apr 15, 2024
3777bf3
Merge branch 'develop' into CYB-183
vpavlenko-cv Apr 20, 2024
dddcc68
Merge branch 'develop' into CYB-183
stas-panasiuk Apr 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 9 additions & 6 deletions ...ain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/AvroParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,17 +12,12 @@

package com.cloudera.parserchains.parsers;

import static com.cloudera.parserchains.core.Constants.DEFAULT_INPUT_FIELD;
import static java.lang.String.format;
import com.cloudera.parserchains.core.FieldName;
import com.cloudera.parserchains.core.FieldValue;
import com.cloudera.parserchains.core.Message;
import com.cloudera.parserchains.core.Parser;
import com.cloudera.parserchains.core.catalog.Configurable;
import com.cloudera.parserchains.core.catalog.MessageParser;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.Optional;
import lombok.extern.slf4j.Slf4j;
import org.apache.avro.AvroRuntimeException;
import org.apache.avro.Schema;
Expand All @@ -35,6 +30,13 @@
import org.apache.flink.core.fs.FileSystem;
import org.apache.flink.core.fs.Path;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.Optional;

import static com.cloudera.parserchains.core.Constants.DEFAULT_INPUT_FIELD;
import static java.lang.String.format;

@MessageParser(
name = "Simple Avro parser",
description = "Parses Avro data by creating a field for each Avro element.")
Expand All @@ -55,7 +57,8 @@ public AvroParser() {
key = "input",
label = "Input Field",
description = "The input field to parse. Default value: '" + DEFAULT_INPUT_FIELD + "'",
defaultValue = DEFAULT_INPUT_FIELD)
defaultValue = DEFAULT_INPUT_FIELD,
isOutputName = true)
public AvroParser inputField(String fieldName) {
if (StringUtils.isNotBlank(fieldName)) {
this.inputField = FieldName.of(fieldName);
Expand Down
11 changes: 7 additions & 4 deletions .../parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/CsvTextParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,6 @@

package com.cloudera.parserchains.parsers;

import static com.cloudera.parserchains.core.utils.StringUtils.getFirstChar;
import static com.cloudera.parserchains.core.utils.StringUtils.unescapeJava;
import static java.lang.String.format;
import com.cloudera.parserchains.core.Constants;
import com.cloudera.parserchains.core.FieldName;
import com.cloudera.parserchains.core.FieldValue;
Expand All @@ -29,13 +26,18 @@
import com.fasterxml.jackson.databind.ObjectReader;
import com.fasterxml.jackson.dataformat.csv.CsvMapper;
import com.fasterxml.jackson.dataformat.csv.CsvSchema;
import org.apache.commons.lang3.StringUtils;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Supplier;
import org.apache.commons.lang3.StringUtils;

import static com.cloudera.parserchains.core.utils.StringUtils.getFirstChar;
import static com.cloudera.parserchains.core.utils.StringUtils.unescapeJava;
import static java.lang.String.format;

/**
* Parses delimited text like CSV.
Expand Down Expand Up @@ -95,6 +97,7 @@ public CsvTextParser withInputField(FieldName inputField) {
@Configurable(key = "inputField",
label = "Input Field",
description = "The name of the input field to parse. Default value: '" + Constants.DEFAULT_INPUT_FIELD + "'",
isOutputName = true,
defaultValue = Constants.DEFAULT_INPUT_FIELD)
public CsvTextParser withInputField(String fieldName) {
if (StringUtils.isNotEmpty(fieldName)) {
Expand Down
9 changes: 6 additions & 3 deletions ...ains-parsers/src/main/java/com/cloudera/parserchains/parsers/DelimitedKeyValueParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@

package com.cloudera.parserchains.parsers;

import static java.lang.String.format;
import com.cloudera.parserchains.core.Constants;
import com.cloudera.parserchains.core.FieldName;
import com.cloudera.parserchains.core.FieldValue;
Expand All @@ -22,10 +21,13 @@
import com.cloudera.parserchains.core.StringFieldValue;
import com.cloudera.parserchains.core.catalog.Configurable;
import com.cloudera.parserchains.core.catalog.MessageParser;
import java.util.Optional;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;

import java.util.Optional;

import static java.lang.String.format;

@MessageParser(
name="Delimited Key Values",
description="Parses delimited key-value pairs."
Expand Down Expand Up @@ -57,7 +59,8 @@ public DelimitedKeyValueParser() {
key="input",
label="Input Field",
description="The input field to parse. Default value: '" + Constants.DEFAULT_INPUT_FIELD + "'",
defaultValue=Constants.DEFAULT_INPUT_FIELD)
defaultValue=Constants.DEFAULT_INPUT_FIELD,
isOutputName = true)
public DelimitedKeyValueParser inputField(String inputField) {
if(StringUtils.isNotBlank(inputField)) {
this.inputField = FieldName.of(inputField);
Expand Down
9 changes: 6 additions & 3 deletions ...ain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/GrokParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@

package com.cloudera.parserchains.parsers;

import static java.lang.String.format;
import com.cloudera.parserchains.core.Constants;
import com.cloudera.parserchains.core.FieldName;
import com.cloudera.parserchains.core.Message;
Expand All @@ -23,11 +22,14 @@
import com.cloudera.parserchains.core.catalog.WidgetType;
import io.krakens.grok.api.Grok;
import io.krakens.grok.api.GrokCompiler;
import org.apache.commons.lang3.StringUtils;

import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.apache.commons.lang3.StringUtils;

import static java.lang.String.format;

@MessageParser(
name = "Grok",
Expand Down Expand Up @@ -74,7 +76,8 @@ private void doParse(String textToParse, Message.Builder output) {

@Configurable(key = "grokPattern",
description = "Define a Grok pattern that can be referenced from an expression.",
orderPriority = 1)
orderPriority = 1,
isOutputName = true)
public GrokParser pattern(
@Parameter(key = "name", label = "Pattern Name") String patternName,
@Parameter(key = "regex", label = "Pattern Regex", widgetType = WidgetType.TEXTAREA) String patternRegex) {
Expand Down
11 changes: 7 additions & 4 deletions ...ain/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/JSONParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,6 @@

package com.cloudera.parserchains.parsers;

import static com.cloudera.parserchains.core.Constants.DEFAULT_INPUT_FIELD;
import static java.lang.String.format;
import com.cloudera.parserchains.core.FieldName;
import com.cloudera.parserchains.core.Message;
import com.cloudera.parserchains.core.Parser;
Expand All @@ -23,6 +21,8 @@
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import org.apache.commons.lang3.StringUtils;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
Expand All @@ -31,7 +31,9 @@
import java.util.Objects;
import java.util.Optional;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;

import static com.cloudera.parserchains.core.Constants.DEFAULT_INPUT_FIELD;
import static java.lang.String.format;

@MessageParser(
name="Simple JSON",
Expand All @@ -52,7 +54,8 @@ public JSONParser() {
key="input",
label="Input Field",
description= "The input field to parse. Default value: '" + DEFAULT_INPUT_FIELD + "'",
defaultValue=DEFAULT_INPUT_FIELD)
defaultValue=DEFAULT_INPUT_FIELD,
isOutputName = true)
public JSONParser inputField(String fieldName) {
if(StringUtils.isNotBlank(fieldName)) {
this.inputField = FieldName.of(fieldName);
Expand Down
17 changes: 10 additions & 7 deletions .../parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/StellarParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@

package com.cloudera.parserchains.parsers;

import static com.cloudera.parserchains.core.Constants.DEFAULT_INPUT_FIELD;
import com.cloudera.cyber.parser.MessageToParse;
import com.cloudera.cyber.stellar.MetronCompatibilityParser;
import com.cloudera.parserchains.core.FieldName;
Expand All @@ -21,11 +20,6 @@
import com.cloudera.parserchains.core.Parser;
import com.cloudera.parserchains.core.catalog.Configurable;
import com.cloudera.parserchains.core.catalog.MessageParser;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
Expand All @@ -35,6 +29,14 @@
import org.apache.metron.parsers.interfaces.MessageParserResult;
import org.json.simple.JSONObject;

import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;

import static com.cloudera.parserchains.core.Constants.DEFAULT_INPUT_FIELD;

@MessageParser(
name = "Metron Stellar parser",
description = "Metron compatibility parser.")
Expand All @@ -53,7 +55,8 @@ public StellarParser() {
key = "input",
label = "Input Field",
description = "The input field to parse. Default value: '" + DEFAULT_INPUT_FIELD + "'",
defaultValue = DEFAULT_INPUT_FIELD)
defaultValue = DEFAULT_INPUT_FIELD,
isOutputName = true)
public StellarParser inputField(String fieldName) {
if (StringUtils.isNotBlank(fieldName)) {
this.inputField = FieldName.of(fieldName);
Expand Down
9 changes: 6 additions & 3 deletions ...n/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/SyslogParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@

package com.cloudera.parserchains.parsers;

import static java.lang.String.format;
import com.cloudera.parserchains.core.Constants;
import com.cloudera.parserchains.core.FieldName;
import com.cloudera.parserchains.core.Message;
Expand All @@ -22,9 +21,12 @@
import com.cloudera.parserchains.core.catalog.MessageParser;
import com.github.palindromicity.syslog.SyslogParserBuilder;
import com.github.palindromicity.syslog.SyslogSpecification;
import java.util.Objects;
import org.apache.commons.lang3.StringUtils;

import java.util.Objects;

import static java.lang.String.format;

@MessageParser(
name="Syslog",
description="Parses Syslog according to RFC 3164 and 5424.")
Expand Down Expand Up @@ -62,7 +64,8 @@ public SyslogSpecification getSpecification() {
@Configurable(key="inputField",
label="Input Field",
description="The name of the input field to parse. Default value: '" + Constants.DEFAULT_INPUT_FIELD + "'",
defaultValue = Constants.DEFAULT_INPUT_FIELD)
defaultValue = Constants.DEFAULT_INPUT_FIELD,
isOutputName = true)
public SyslogParser withInputField(String inputField) {
if(StringUtils.isNotBlank(inputField)) {
this.inputField = FieldName.of(inputField);
Expand Down
8 changes: 5 additions & 3 deletions ...chains-parsers/src/main/java/com/cloudera/parserchains/parsers/TimestampFormatParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@

package com.cloudera.parserchains.parsers;

import static java.util.stream.Collectors.toList;
import com.cloudera.parserchains.core.FieldName;
import com.cloudera.parserchains.core.FieldValue;
import com.cloudera.parserchains.core.Message;
Expand All @@ -21,6 +20,8 @@
import com.cloudera.parserchains.core.catalog.Configurable;
import com.cloudera.parserchains.core.catalog.MessageParser;
import com.cloudera.parserchains.core.catalog.Parameter;
import org.apache.commons.lang3.StringUtils;

import java.io.Serializable;
import java.time.DateTimeException;
import java.time.Instant;
Expand All @@ -32,7 +33,8 @@
import java.util.Iterator;
import java.util.List;
import java.util.TimeZone;
import org.apache.commons.lang3.StringUtils;

import static java.util.stream.Collectors.toList;

/**
* A parser to extract formatted timestamps and express them as epoch time, preserving the
Expand Down Expand Up @@ -82,7 +84,7 @@ private Long parseDate(String inputValue, DateTimeFormatter format, String tz) t
description = "The field that will contain the timestamp.",
multipleValues = true)
public TimestampFormatParser withOutputField(
@Parameter(key = "field", label = "Input Field", description = "Field to be parsed", required = true) String fieldName,
@Parameter(key = "field", label = "Input Field", description = "Field to be parsed", required = true, isOutputName = true) String fieldName,
@Parameter(key = "format", label = "Time format", description = "A compatible time format", required = true) String format,
@Parameter(key = "tz", label = "Timezone", description = "Optionally set the expected timezone", required = true) String tz
) {
Expand Down
15 changes: 9 additions & 6 deletions ...n/parser-chains-parsers/src/main/java/com/cloudera/parserchains/parsers/XMLFlattener.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,6 @@

package com.cloudera.parserchains.parsers;

import static java.lang.String.format;
import static org.apache.commons.lang3.StringUtils.isNotBlank;
import com.cloudera.parserchains.core.Constants;
import com.cloudera.parserchains.core.FieldName;
import com.cloudera.parserchains.core.FieldValue;
Expand All @@ -23,16 +21,20 @@
import com.cloudera.parserchains.core.catalog.Configurable;
import com.cloudera.parserchains.core.catalog.MessageParser;
import com.github.wnameless.json.flattener.JsonFlattener;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.XML;
import org.json.XMLParserConfiguration;

import java.util.Map;
import java.util.Objects;
import java.util.Optional;

import static java.lang.String.format;
import static org.apache.commons.lang3.StringUtils.isNotBlank;

@MessageParser(
name = "XML Flattener",
description = "Flattens XML data."
Expand All @@ -53,7 +55,8 @@ public XMLFlattener() {
@Configurable(key = "inputField",
label = "Input Field",
description = "The name of the input field to parse. Default value: '" + Constants.DEFAULT_INPUT_FIELD + "'",
defaultValue = Constants.DEFAULT_INPUT_FIELD)
defaultValue = Constants.DEFAULT_INPUT_FIELD,
isOutputName = true)
public XMLFlattener inputField(String fieldName) {
if (StringUtils.isNotEmpty(fieldName)) {
this.inputField = FieldName.of(fieldName);
Expand Down