From e56b851d1596256cd71fd3d3a3dcc18a8232dee7 Mon Sep 17 00:00:00 2001 From: ppom <> Date: Wed, 4 Oct 2023 12:00:00 +0200 Subject: [PATCH] support json, jsonnet, yaml formats - jsonnet, json and yaml support for configuration - json and yaml support for output formats fix #40 fix #27 --- README.md | 57 ++++++++++++++++++++--- app/client.go | 76 ++++++++++--------------------- app/main.go | 41 +++++++++-------- app/reaction.yml | 8 ++-- app/startup.go | 15 ++++-- app/types.go | 48 ++++++++++---------- config/reaction.jsonnet | 88 ++++++++++++++++++++++++++++++++++++ config/reaction.test.jsonnet | 30 ++++++++++++ config/reaction.test.yml | 2 +- go.mod | 5 +- go.sum | 12 +++-- 11 files changed, 269 insertions(+), 113 deletions(-) create mode 100644 config/reaction.jsonnet create mode 100644 config/reaction.test.jsonnet diff --git a/README.md b/README.md index 2baa8a1..b77d08b 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ and takes action, such as banning ips. ## rationale -i was using fail2ban since quite a long time, but i was a bit frustrated by it's cpu consumption +i was using fail2ban since quite a long time, but i was a bit frustrated by its cpu consumption and all its heavy default configuration. in my view, a security-oriented program should be simple to configure (`sudo` is a very bad example!) @@ -20,7 +20,7 @@ and an always-running daemon should be implemented in a fast language. ## configuration -this configuration file is all that should be needed to prevent bruteforce attacks on an ssh server. +this configuration file is all that should be needed to prevent brute force attacks on an ssh server. see [reaction.service](./config/reaction.service) and [reaction.yml](./app/reaction.yml) for the fully explained examples. @@ -32,6 +32,9 @@ definitions: patterns: ip: '(([0-9]{1,3}\.){3}[0-9]{1,3})|([0-9a-fA-F:]{2,90})' + ignore: + - '127.0.0.1' + - '::1' streams: ssh: @@ -39,17 +42,57 @@ streams: filters: failedlogin: regex: - - authentication failure;.*rhost= + - 'authentication failure;.*rhost=' retry: 3 - retry-period: 6h + retryperiod: '6h' actions: ban: cmd: *iptablesban unban: - cmd: *iptablesunban - after: 48h + cmd: *iptablesunban + after: '48h' ``` +jsonnet is also supported: + +`/etc/reaction.jsonnet` +```jsonnet +local iptablesban = ['iptables', '-w', '-A', 'reaction', '1', '-s', '', '-j', 'DROP']; +local iptablesunban = ['iptables', '-w', '-D', 'reaction', '1', '-s', '', '-j', 'DROP']; +{ + patterns: { + ip: { + regex: @'(?:(?:[0-9]{1,3}\.){3}[0-9]{1,3})|(?:[0-9a-fA-F:]{2,90})', + ignore: ['127.0.0.1', '::1'], + }, + }, + streams: { + ssh: { + cmd: ['journalctl', '-fu', 'sshd.service'], + filters: { + failedlogin: { + regex: [ @'authentication failure;.*rhost=' ], + retry: 3, + retryperiod: '6h', + actions: { + ban: { + cmd: iptablesban, + }, + unban: { + cmd: iptablesunban, + after: '48h', + onexit: true, + }, + }, + }, + }, + }, + }, +} +``` + +note that both yaml and jsonnet are extensions of json, so it is also inherently supported. + `/etc/systemd/system/reaction.service` ```systemd [Unit] @@ -92,7 +135,7 @@ $ go build . in addition to the [package](https://framagit.org/ppom/nixos/-/blob/cf5448b21ae3386265485308a6cd077e8068ad77/pkgs/reaction/default.nix) and [module](https://framagit.org/ppom/nixos/-/blob/cf5448b21ae3386265485308a6cd077e8068ad77/modules/common/reaction.nix) -that i didn't tried to upstream to nixpkgs yet (although they are ready), i use extensively reaction on my servers. if you're using nixos, +that i didn't try to upstream to nixpkgs yet (although they are ready), i use extensively reaction on my servers. if you're using nixos, consider reading and building upon [my own building blocks](https://framagit.org/ppom/nixos/-/blob/cf5448b21ae3386265485308a6cd077e8068ad77/modules/common/reaction-variables.nix), [my own non-root reaction conf, including conf for SSH, port scanning & Nginx common attack URLS](https://framagit.org/ppom/nixos/-/blob/cf5448b21ae3386265485308a6cd077e8068ad77/modules/common/reaction-custom.nix), and the configuration for [nextcloud](https://framagit.org/ppom/nixos/-/blob/cf5448b21ae3386265485308a6cd077e8068ad77/modules/musi/file.ppom.me.nix#L53), diff --git a/app/client.go b/app/client.go index de1b929..e39db94 100644 --- a/app/client.go +++ b/app/client.go @@ -3,13 +3,14 @@ package app import ( "bufio" "encoding/gob" + "encoding/json" "fmt" "log" "net" "os" "regexp" - "gopkg.in/yaml.v3" + "sigs.k8s.io/yaml" ) const ( @@ -48,8 +49,8 @@ func SendAndRetrieve(data Request) Response { } type PatternStatus struct { - Matches int `yaml:"matches"` - Actions map[string][]string `yaml:"actions"` + Matches int `json:"matches,omitempty"` + Actions map[string][]string `json:"actions,omitempty"` } type MapPatternStatus map[string]*PatternStatus type MapPatternStatusFlush MapPatternStatus @@ -57,53 +58,14 @@ type MapPatternStatusFlush MapPatternStatus type ClientStatus map[string]map[string]MapPatternStatus type ClientStatusFlush ClientStatus -// This block is made to hide pending_actions when empty -// and matches_since_last_trigger when zero -type FullPatternStatus PatternStatus -type MatchesStatus struct { - Matches int `yaml:"matches"` -} -type ActionsStatus struct { - Actions map[string][]string `yaml:"actions"` -} - -func (mps MapPatternStatus) MarshalYAML() (interface{}, error) { - ret := make(map[string]interface{}) - for k, v := range mps { - if v.Matches == 0 { - if len(v.Actions) != 0 { - ret[k] = ActionsStatus{v.Actions} - } - } else { - if len(v.Actions) != 0 { - ret[k] = v - } else { - ret[k] = MatchesStatus{v.Matches} - } - } - } - return ret, nil -} - -func (mps MapPatternStatusFlush) MarshalYAML() (interface{}, error) { - var ret interface{} +func (mps MapPatternStatusFlush) MarshalJSON() ([]byte, error) { for _, v := range mps { - if v.Matches == 0 { - if len(v.Actions) != 0 { - ret = ActionsStatus{v.Actions} - } - } else { - if len(v.Actions) != 0 { - ret = v - } else { - ret = MatchesStatus{v.Matches} - } - } + return json.Marshal(v) } - return ret, nil + return []byte(""), nil } -func (csf ClientStatusFlush) MarshalYAML() (interface{}, error) { +func (csf ClientStatusFlush) MarshalJSON() ([]byte, error) { ret := make(map[string]map[string]MapPatternStatusFlush) for k, v := range csf { ret[k] = make(map[string]MapPatternStatusFlush) @@ -111,7 +73,7 @@ func (csf ClientStatusFlush) MarshalYAML() (interface{}, error) { ret[k][kk] = MapPatternStatusFlush(vv) } } - return ret, nil + return json.Marshal(ret) } // end block @@ -122,13 +84,19 @@ func usage(err string) { log.Fatalln(err) } -func ClientShow(streamfilter string) { +func ClientShow(streamfilter, format string) { response := SendAndRetrieve(Request{Show, streamfilter}) if response.Err != nil { log.Fatalln("Received error from daemon:", response.Err) os.Exit(1) } - text, err := yaml.Marshal(response.ClientStatus) + var text []byte + var err error + if format == "json" { + text, err = json.MarshalIndent(response.ClientStatus, "", " ") + } else { + text, err = yaml.Marshal(response.ClientStatus) + } if err != nil { log.Fatalln("Failed to convert daemon binary response to text format:", err) } @@ -136,13 +104,19 @@ func ClientShow(streamfilter string) { os.Exit(0) } -func ClientFlush(pattern, streamfilter string) { +func ClientFlush(pattern, streamfilter, format string) { response := SendAndRetrieve(Request{Flush, pattern}) if response.Err != nil { log.Fatalln("Received error from daemon:", response.Err) os.Exit(1) } - text, err := yaml.Marshal(ClientStatusFlush(response.ClientStatus)) + var text []byte + var err error + if format == "json" { + text, err = json.MarshalIndent(ClientStatusFlush(response.ClientStatus), "", " ") + } else { + text, err = yaml.Marshal(ClientStatusFlush(response.ClientStatus)) + } if err != nil { log.Fatalln("Failed to convert daemon binary response to text format:", err) } diff --git a/app/main.go b/app/main.go index e516dff..899d7c8 100644 --- a/app/main.go +++ b/app/main.go @@ -57,8 +57,7 @@ func subCommandParse(f *flag.FlagSet, maxRemainingArgs int) { } // FIXME add this options for show & flush -// -l/--limit .STREAM[.FILTER] # limit to stream and filter -// -f/--format yaml|json # (default: yaml) +// -l/--limit .STREAM[.FILTER] # limit to stream and filter func basicUsage() { const ( bold = "\033[1m" @@ -70,9 +69,9 @@ func basicUsage() { # start the daemon # options: - -c/--config CONFIG_FILE # configuration file (required) - -s/--socket SOCKET # path to the client-daemon communication socket - # (default: /run/reaction/reaction.sock) + -c/--config CONFIG_FILE # configuration file in json, jsonnet or yaml format (required) + -s/--socket SOCKET # path to the client-daemon communication socket + # (default: /run/reaction/reaction.sock) ` + bold + `reaction example-conf` + reset + ` # print a configuration file example @@ -82,17 +81,19 @@ func basicUsage() { # (e.g know what is currenly banned) # options: - -s/--socket SOCKET # path to the client-daemon communication socket + -s/--socket SOCKET # path to the client-daemon communication socket + -f/--format yaml|json # (default: yaml) ` + bold + `reaction flush` + reset + ` TARGET # run currently active matches and pending actions for the specified TARGET # (then show flushed matches and actions) # options: - -s/--socket SOCKET # path to the client-daemon communication socket + -s/--socket SOCKET # path to the client-daemon communication socket + -f/--format yaml|json # (default: yaml) -` + bold + `reaction test-regex` + reset + ` REGEX LINE # test REGEX against LINE -cat FILE | ` + bold + `reaction test-regex` + reset + ` REGEX # test REGEX against each line of FILE +` + bold + `reaction test-regex` + reset + ` REGEX LINE # test REGEX against LINE +cat FILE | ` + bold + `reaction test-regex` + reset + ` REGEX # test REGEX against each line of FILE `) } @@ -133,13 +134,9 @@ func Main() { queryFormat := addFormatFlag(f) limit := addLimitFlag(f) subCommandParse(f, 0) - // if *queryFormat != "yaml" && *queryFormat != "json" { - // fmt.Println("only `yaml` and `json` formats are supported.") - // f.PrintDefaults() - // os.Exit(1) - // } - if *queryFormat != "yaml" { - fmt.Println("for now, only `yaml` format is supported.") + if *queryFormat != "yaml" && *queryFormat != "json" { + fmt.Println("only yaml and json formats are supported") + f.PrintDefaults() os.Exit(1) } if *limit != "" { @@ -147,12 +144,18 @@ func Main() { os.Exit(1) } // f.Arg(0) is "" if there is no remaining argument - ClientShow(*limit) + ClientShow(*limit, *queryFormat) case "flush": SocketPath = addSocketFlag(f) + queryFormat := addFormatFlag(f) limit := addLimitFlag(f) subCommandParse(f, 1) + if *queryFormat != "yaml" && *queryFormat != "json" { + fmt.Println("only yaml and json formats are supported") + f.PrintDefaults() + os.Exit(1) + } if f.Arg(0) == "" { fmt.Println("subcommand flush takes one TARGET argument") basicUsage() @@ -162,7 +165,7 @@ func Main() { fmt.Println("for now, -l/--limit is not supported") os.Exit(1) } - ClientFlush(f.Arg(0), f.Arg(1)) + ClientFlush(f.Arg(0), *limit, *queryFormat) case "test-regex": // socket not needed, no interaction with the daemon @@ -178,7 +181,7 @@ func Main() { os.Exit(1) } if f.Arg(1) == "" { - fmt.Println("INFO no second argument. reading from stdin.") + fmt.Println("INFO no second argument: reading from stdin") MatchStdin(regex) } else { diff --git a/app/reaction.yml b/app/reaction.yml index e8d5e8b..9f20cec 100644 --- a/app/reaction.yml +++ b/app/reaction.yml @@ -35,12 +35,12 @@ streams: # is predefined in the patterns section # ip's regex is inserted in the following regex - authentication failure;.*rhost= - # if retry and retry-period are defined, + # if retry and retryperiod are defined, # the actions will only take place if a same pattern is - # found `retry` times in a `retry-period` interval + # found `retry` times in a `retryperiod` interval retry: 3 # format is defined here: https://pkg.go.dev/time#ParseDuration - retry-period: 6h + retryperiod: 6h # actions are run by the filter when regexes are matched actions: # actions have a user-defined name @@ -50,7 +50,7 @@ streams: unban: cmd: *iptablesunban # if after is defined, the action will not take place immediately, but after a specified duration - # same format as retry-period + # same format as retryperiod after: 48h # let's say reaction is quitting. does it run all those pending commands which had an `after` duration set? # if you want reaction to run those pending commands before exiting, you can set this: diff --git a/app/startup.go b/app/startup.go index bcafd31..6ebab7b 100644 --- a/app/startup.go +++ b/app/startup.go @@ -1,6 +1,7 @@ package app import ( + "encoding/json" "fmt" "log" "os" @@ -8,7 +9,7 @@ import ( "strings" "time" - "gopkg.in/yaml.v3" + "github.com/google/go-jsonnet" ) func (c *Conf) setup() { @@ -131,13 +132,21 @@ func (c *Conf) setup() { func parseConf(filename string) *Conf { - data, err := os.ReadFile(filename) + data, err := os.Open(filename) if err != nil { log.Fatalln("FATAL Failed to read configuration file:", err) } var conf Conf - err = yaml.Unmarshal(data, &conf) + if filename[len(filename)-4:] == ".yml" || filename[len(filename)-5:] == ".yaml" { + err = jsonnet.NewYAMLToJSONDecoder(data).Decode(&conf) + } else { + var jsondata string + jsondata, err = jsonnet.MakeVM().EvaluateFile(filename) + if err == nil { + err = json.Unmarshal([]byte(jsondata), &conf) + } + } if err != nil { log.Fatalln("FATAL Failed to parse configuration file:", err) } diff --git a/app/types.go b/app/types.go index cae09c0..9a297da 100644 --- a/app/types.go +++ b/app/types.go @@ -8,54 +8,54 @@ import ( ) type Conf struct { - Patterns map[string]*Pattern `yaml:"patterns"` - Streams map[string]*Stream `yaml:"streams"` + Patterns map[string]*Pattern `json:"patterns"` + Streams map[string]*Stream `json:"streams"` } type Pattern struct { - Regex string `yaml:"regex"` - Ignore []string `yaml:"ignore"` + Regex string `json:"regex"` + Ignore []string `json:"ignore"` - name string `yaml:"-"` - nameWithBraces string `yaml:"-"` + name string `json:"-"` + nameWithBraces string `json:"-"` } // Stream, Filter & Action structures must never be copied. // They're always referenced through pointers type Stream struct { - name string `yaml:"-"` + name string `json:"-"` - Cmd []string `yaml:"cmd"` - Filters map[string]*Filter `yaml:"filters"` + Cmd []string `json:"cmd"` + Filters map[string]*Filter `json:"filters"` } type Filter struct { - stream *Stream `yaml:"-"` - name string `yaml:"-"` + stream *Stream `json:"-"` + name string `json:"-"` - Regex []string `yaml:"regex"` - compiledRegex []regexp.Regexp `yaml:"-"` - pattern *Pattern `yaml:"-"` + Regex []string `json:"regex"` + compiledRegex []regexp.Regexp `json:"-"` + pattern *Pattern `json:"-"` - Retry int `yaml:"retry"` - RetryPeriod string `yaml:"retry-period"` - retryDuration time.Duration `yaml:"-"` + Retry int `json:"retry"` + RetryPeriod string `json:"retryperiod"` + retryDuration time.Duration `json:"-"` - Actions map[string]*Action `yaml:"actions"` + Actions map[string]*Action `json:"actions"` longuestActionDuration *time.Duration } type Action struct { - filter *Filter `yaml:"-"` - name string `yaml:"-"` + filter *Filter `json:"-"` + name string `json:"-"` - Cmd []string `yaml:"cmd"` + Cmd []string `json:"cmd"` - After string `yaml:"after"` - afterDuration time.Duration `yaml:"-"` + After string `json:"after"` + afterDuration time.Duration `json:"-"` - OnExit bool `yaml:"onexit"` + OnExit bool `json:"onexit"` } type LogEntry struct { diff --git a/config/reaction.jsonnet b/config/reaction.jsonnet new file mode 100644 index 0000000..252d3b3 --- /dev/null +++ b/config/reaction.jsonnet @@ -0,0 +1,88 @@ +// This file is using JSONNET, a complete configuration language based on JSON +// See https://jsonnet.org +// JSONNET is a superset of JSON, so one can write plain JSON files if wanted. + +// variables defined for later use. +local iptablesban = ['iptables', '-w', '-A', 'reaction', '1', '-s', '', '-j', 'DROP']; +local iptablesunban = ['iptables', '-w', '-D', 'reaction', '1', '-s', '', '-j', 'DROP']; + +{ + // patterns are substitued in regexes. + // when a filter performs an action, it replaces the found pattern + patterns: { + ip: { + // reaction regex syntax is defined here: https://github.com/google/re2/wiki/Syntax + // jsonnet's @'string' is for verbatim strings + regex: @'(?:(?:[0-9]{1,3}\.){3}[0-9]{1,3})|(?:[0-9a-fA-F:]{2,90})', + ignore: ['127.0.0.1', '::1'], + }, + }, + + // streams are commands + // they're run and their ouptut is captured + // *example:* `tail -f /var/log/nginx/access.log` + // their output will be used by one or more filters + streams: { + // streams have a user-defined name + ssh: { + // note that if the command is not in environment's `PATH` + // its full path must be given. + cmd: ['journalctl', '-fu', 'sshd.service'], + // filters run actions when they match regexes on a stream + filters: { + // filters have a user-defined name + failedlogin: { + // reaction's regex syntax is defined here: https://github.com/google/re2/wiki/Syntax + regex: [ + // is predefined in the patterns section + // ip's regex is inserted in the following regex + 'authentication failure;.*rhost=', + ], + // if retry and retryperiod are defined, + // the actions will only take place if a same pattern is + // found `retry` times in a `retryperiod` interval + retry: 3, + // format is defined here: https://pkg.go.dev/time#ParseDuration + retryperiod: '6h', + // actions are run by the filter when regexes are matched + actions: { + // actions have a user-defined name + ban: { + // JSONNET substitutes the variable (defined at the beginning of the file) + cmd: iptablesban, + }, + unban: { + cmd: iptablesunban, + // if after is defined, the action will not take place immediately, but after a specified duration + // same format as retryperiod + after: '48h', + // let's say reaction is quitting. does it run all those pending commands which had an `after` duration set? + // if you want reaction to run those pending commands before exiting, you can set this: + onexit: true, + // (defaults to false) + // here it is not useful because we will flush the chain containing the bans anyway + // (see /conf/reaction.service) + }, + }, + }, + }, + }, + }, +} + +// persistence + +// tldr; when an `after` action is set in a filter, such filter acts as a 'jail', +// which is persisted after reboots. + +// full; +// when a filter is triggered, there are 2 flows: +// +// if none of its actions have an `after` directive set: +// no action will be replayed. +// +// else (if at least one action has an `after` directive set): +// if reaction stops while `after` actions are pending: +// and reaction starts again while those actions would still be pending: +// reaction executes the past actions (actions without after or with then+after < now) +// and plans the execution of future actions (actions with then+after > now) diff --git a/config/reaction.test.jsonnet b/config/reaction.test.jsonnet new file mode 100644 index 0000000..d6c6cfe --- /dev/null +++ b/config/reaction.test.jsonnet @@ -0,0 +1,30 @@ +{ + patterns: { + num: { + regex: '[0-9]+', + }, + }, + + streams: { + tailDown1: { + cmd: ['sh', '-c', "echo 1 2 3 4 5 1 2 3 4 5 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 | tr ' ' '\n' | while read i; do sleep 2; echo found $(($i % 10)); done"], + filters: { + findIP: { + regex: ['^found $'], + retry: 3, + retryperiod: '30s', + actions: { + damn: { + cmd: ['echo', ''], + }, + undamn: { + cmd: ['echo', 'undamn', ''], + after: '30s', + onexit: true, + }, + }, + }, + }, + }, + }, +} diff --git a/config/reaction.test.yml b/config/reaction.test.yml index 89d8fbc..b5cf9f0 100644 --- a/config/reaction.test.yml +++ b/config/reaction.test.yml @@ -15,7 +15,7 @@ streams: regex: - '^found $' retry: 3 - retry-period: 30s + retryperiod: 30s actions: damn: cmd: [ "echo", "" ] diff --git a/go.mod b/go.mod index ccd1bc3..89bb599 100644 --- a/go.mod +++ b/go.mod @@ -3,5 +3,8 @@ module framagit.org/ppom/reaction go 1.19 require ( - gopkg.in/yaml.v3 v3.0.1 + github.com/google/go-jsonnet v0.20.0 + sigs.k8s.io/yaml v1.1.0 ) + +require gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index a62c313..fe8df87 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,10 @@ -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-jsonnet v0.20.0 h1:WG4TTSARuV7bSm4PMB4ohjxe33IHT5WVTrJSU33uT4g= +github.com/google/go-jsonnet v0.20.0/go.mod h1:VbgWF9JX7ztlv770x/TolZNGGFfiHEVx9G6ca2eUmeA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=