support json, jsonnet, yaml formats

- jsonnet, json and yaml support for configuration
- json and yaml support for output formats

fix #40
fix #27

README.md

@@ -10,7 +10,7 @@ and takes action, such as banning ips.
 
 ## rationale
 
-i was using fail2ban since quite a long time, but i was a bit frustrated by it's cpu consumption
+i was using fail2ban since quite a long time, but i was a bit frustrated by its cpu consumption
 and all its heavy default configuration.
 
 in my view, a security-oriented program should be simple to configure (`sudo` is a very bad example!)
@@ -32,6 +32,9 @@ definitions:
 
 patterns:
   ip: '(([0-9]{1,3}\.){3}[0-9]{1,3})|([0-9a-fA-F:]{2,90})'
+  ignore:
+    - '127.0.0.1'
+    - '::1'
 
 streams:
   ssh:
@@ -39,17 +42,57 @@ streams:
     filters:
       failedlogin:
         regex:
-          - authentication failure;.*rhost=<ip>
+          - 'authentication failure;.*rhost=<ip>'
         retry: 3
-        retry-period: 6h
+        retryperiod: '6h'
         actions:
           ban:
             cmd: *iptablesban
           unban:
             cmd: *iptablesunban
-            after: 48h
+            after: '48h'
 ```
 
+jsonnet is also supported:
+
+`/etc/reaction.jsonnet`
+```jsonnet
+local iptablesban = ['iptables', '-w', '-A', 'reaction', '1', '-s', '<ip>', '-j', 'DROP'];
+local iptablesunban = ['iptables', '-w', '-D', 'reaction', '1', '-s', '<ip>', '-j', 'DROP'];
+{
+  patterns: {
+    ip: {
+      regex: @'(?:(?:[0-9]{1,3}\.){3}[0-9]{1,3})|(?:[0-9a-fA-F:]{2,90})',
+      ignore: ['127.0.0.1', '::1'],
+    },
+  },
+  streams: {
+    ssh: {
+      cmd: ['journalctl', '-fu', 'sshd.service'],
+      filters: {
+        failedlogin: {
+          regex: [ @'authentication failure;.*rhost=<ip>' ],
+          retry: 3,
+          retryperiod: '6h',
+          actions: {
+            ban: {
+              cmd: iptablesban,
+            },
+            unban: {
+              cmd: iptablesunban,
+              after: '48h',
+              onexit: true,
+            },
+          },
+        },
+      },
+    },
+  },
+}
+```
+
+note that both yaml and jsonnet are extensions of json, so it is also inherently supported.
+
 `/etc/systemd/system/reaction.service`
 ```systemd
 [Unit]
@@ -92,7 +135,7 @@ $ go build .
 
 in addition to the [package](https://framagit.org/ppom/nixos/-/blob/cf5448b21ae3386265485308a6cd077e8068ad77/pkgs/reaction/default.nix)
 and [module](https://framagit.org/ppom/nixos/-/blob/cf5448b21ae3386265485308a6cd077e8068ad77/modules/common/reaction.nix)
-that i didn't tried to upstream to nixpkgs yet (although they are ready), i use extensively reaction on my servers. if you're using nixos,
+that i didn't try to upstream to nixpkgs yet (although they are ready), i use extensively reaction on my servers. if you're using nixos,
 consider reading and building upon [my own building blocks](https://framagit.org/ppom/nixos/-/blob/cf5448b21ae3386265485308a6cd077e8068ad77/modules/common/reaction-variables.nix),
 [my own non-root reaction conf, including conf for SSH, port scanning & Nginx common attack URLS](https://framagit.org/ppom/nixos/-/blob/cf5448b21ae3386265485308a6cd077e8068ad77/modules/common/reaction-custom.nix),
 and the configuration for [nextcloud](https://framagit.org/ppom/nixos/-/blob/cf5448b21ae3386265485308a6cd077e8068ad77/modules/musi/file.ppom.me.nix#L53),

app/client.go

@@ -3,13 +3,14 @@ package app
 import (
 	"bufio"
 	"encoding/gob"
+	"encoding/json"
 	"fmt"
 	"log"
 	"net"
 	"os"
 	"regexp"
 
-	"gopkg.in/yaml.v3"
+	"sigs.k8s.io/yaml"
 )
 
 const (
@@ -48,8 +49,8 @@ func SendAndRetrieve(data Request) Response {
 }
 
 type PatternStatus struct {
-	Matches int                 `yaml:"matches"`
+	Matches int                 `json:"matches,omitempty"`
-	Actions map[string][]string `yaml:"actions"`
+	Actions map[string][]string `json:"actions,omitempty"`
 }
 type MapPatternStatus map[string]*PatternStatus
 type MapPatternStatusFlush MapPatternStatus
@@ -57,53 +58,14 @@ type MapPatternStatusFlush MapPatternStatus
 type ClientStatus map[string]map[string]MapPatternStatus
 type ClientStatusFlush ClientStatus
 
-// This block is made to hide pending_actions when empty
+func (mps MapPatternStatusFlush) MarshalJSON() ([]byte, error) {
-// and matches_since_last_trigger when zero
-type FullPatternStatus PatternStatus
-type MatchesStatus struct {
-	Matches int `yaml:"matches"`
-}
-type ActionsStatus struct {
-	Actions map[string][]string `yaml:"actions"`
-}
-
-func (mps MapPatternStatus) MarshalYAML() (interface{}, error) {
-	ret := make(map[string]interface{})
-	for k, v := range mps {
-		if v.Matches == 0 {
-			if len(v.Actions) != 0 {
-				ret[k] = ActionsStatus{v.Actions}
-			}
-		} else {
-			if len(v.Actions) != 0 {
-				ret[k] = v
-			} else {
-				ret[k] = MatchesStatus{v.Matches}
-			}
-		}
-	}
-	return ret, nil
-}
-
-func (mps MapPatternStatusFlush) MarshalYAML() (interface{}, error) {
-	var ret interface{}
 	for _, v := range mps {
-		if v.Matches == 0 {
+		return json.Marshal(v)
-			if len(v.Actions) != 0 {
-				ret = ActionsStatus{v.Actions}
 	}
-		} else {
+	return []byte(""), nil
-			if len(v.Actions) != 0 {
-				ret = v
-			} else {
-				ret = MatchesStatus{v.Matches}
-			}
-		}
-	}
-	return ret, nil
 }
 
-func (csf ClientStatusFlush) MarshalYAML() (interface{}, error) {
+func (csf ClientStatusFlush) MarshalJSON() ([]byte, error) {
 	ret := make(map[string]map[string]MapPatternStatusFlush)
 	for k, v := range csf {
 		ret[k] = make(map[string]MapPatternStatusFlush)
@@ -111,7 +73,7 @@ func (csf ClientStatusFlush) MarshalYAML() (interface{}, error) {
 			ret[k][kk] = MapPatternStatusFlush(vv)
 		}
 	}
-	return ret, nil
+	return json.Marshal(ret)
 }
 
 // end block
@@ -122,13 +84,19 @@ func usage(err string) {
 	log.Fatalln(err)
 }
 
-func ClientShow(streamfilter string) {
+func ClientShow(streamfilter, format string) {
 	response := SendAndRetrieve(Request{Show, streamfilter})
 	if response.Err != nil {
 		log.Fatalln("Received error from daemon:", response.Err)
 		os.Exit(1)
 	}
-	text, err := yaml.Marshal(response.ClientStatus)
+	var text []byte
+	var err error
+	if format == "json" {
+		text, err = json.MarshalIndent(response.ClientStatus, "", "  ")
+	} else {
+		text, err = yaml.Marshal(response.ClientStatus)
+	}
 	if err != nil {
 		log.Fatalln("Failed to convert daemon binary response to text format:", err)
 	}
@@ -136,13 +104,19 @@ func ClientShow(streamfilter string) {
 	os.Exit(0)
 }
 
-func ClientFlush(pattern, streamfilter string) {
+func ClientFlush(pattern, streamfilter, format string) {
 	response := SendAndRetrieve(Request{Flush, pattern})
 	if response.Err != nil {
 		log.Fatalln("Received error from daemon:", response.Err)
 		os.Exit(1)
 	}
-	text, err := yaml.Marshal(ClientStatusFlush(response.ClientStatus))
+	var text []byte
+	var err error
+	if format == "json" {
+		text, err = json.MarshalIndent(ClientStatusFlush(response.ClientStatus), "", "  ")
+	} else {
+		text, err = yaml.Marshal(ClientStatusFlush(response.ClientStatus))
+	}
 	if err != nil {
 		log.Fatalln("Failed to convert daemon binary response to text format:", err)
 	}

app/main.go

@@ -58,7 +58,6 @@ func subCommandParse(f *flag.FlagSet, maxRemainingArgs int) {
 
 // FIXME add this options for show & flush
 // -l/--limit .STREAM[.FILTER]      # limit to stream and filter
-// -f/--format yaml|json               # (default: yaml)
 func basicUsage() {
 	const (
 		bold  = "\033[1m"
@@ -70,7 +69,7 @@ func basicUsage() {
   # start the daemon
 
   # options:
-    -c/--config CONFIG_FILE             # configuration file (required)
+    -c/--config CONFIG_FILE          # configuration file in json, jsonnet or yaml format (required)
     -s/--socket SOCKET               # path to the client-daemon communication socket
                                      # (default: /run/reaction/reaction.sock)
 
@@ -83,6 +82,7 @@ func basicUsage() {
 
   # options:
     -s/--socket SOCKET               # path to the client-daemon communication socket
+    -f/--format yaml|json            # (default: yaml)
 
 ` + bold + `reaction flush` + reset + ` TARGET
   # run currently active matches and pending actions for the specified TARGET
@@ -90,6 +90,7 @@ func basicUsage() {
 
   # options:
     -s/--socket SOCKET               # path to the client-daemon communication socket
+    -f/--format yaml|json            # (default: yaml)
 
 ` + bold + `reaction test-regex` + reset + ` REGEX LINE       # test REGEX against LINE
 cat FILE | ` + bold + `reaction test-regex` + reset + ` REGEX # test REGEX against each line of FILE
@@ -133,13 +134,9 @@ func Main() {
 		queryFormat := addFormatFlag(f)
 		limit := addLimitFlag(f)
 		subCommandParse(f, 0)
-		// if *queryFormat != "yaml" && *queryFormat != "json" {
+		if *queryFormat != "yaml" && *queryFormat != "json" {
-		// 	fmt.Println("only `yaml` and `json` formats are supported.")
+			fmt.Println("only yaml and json formats are supported")
-		// 	f.PrintDefaults()
+			f.PrintDefaults()
-		// 	os.Exit(1)
-		// }
-		if *queryFormat != "yaml" {
-			fmt.Println("for now, only `yaml` format is supported.")
 			os.Exit(1)
 		}
 		if *limit != "" {
@@ -147,12 +144,18 @@ func Main() {
 			os.Exit(1)
 		}
 		// f.Arg(0) is "" if there is no remaining argument
-		ClientShow(*limit)
+		ClientShow(*limit, *queryFormat)
 
 	case "flush":
 		SocketPath = addSocketFlag(f)
+		queryFormat := addFormatFlag(f)
 		limit := addLimitFlag(f)
 		subCommandParse(f, 1)
+		if *queryFormat != "yaml" && *queryFormat != "json" {
+			fmt.Println("only yaml and json formats are supported")
+			f.PrintDefaults()
+			os.Exit(1)
+		}
 		if f.Arg(0) == "" {
 			fmt.Println("subcommand flush takes one TARGET argument")
 			basicUsage()
@@ -162,7 +165,7 @@ func Main() {
 			fmt.Println("for now, -l/--limit is not supported")
 			os.Exit(1)
 		}
-		ClientFlush(f.Arg(0), f.Arg(1))
+		ClientFlush(f.Arg(0), *limit, *queryFormat)
 
 	case "test-regex":
 		// socket not needed, no interaction with the daemon
@@ -178,7 +181,7 @@ func Main() {
 			os.Exit(1)
 		}
 		if f.Arg(1) == "" {
-			fmt.Println("INFO  no second argument. reading from stdin.")
+			fmt.Println("INFO  no second argument: reading from stdin")
 
 			MatchStdin(regex)
 		} else {

app/reaction.yml

@@ -35,12 +35,12 @@ streams:
           # <ip> is predefined in the patterns section
           # ip's regex is inserted in the following regex
           - authentication failure;.*rhost=<ip>
-        # if retry and retry-period are defined,
+        # if retry and retryperiod are defined,
         # the actions will only take place if a same pattern is
-        # found `retry` times in a `retry-period` interval
+        # found `retry` times in a `retryperiod` interval
         retry: 3
         # format is defined here: https://pkg.go.dev/time#ParseDuration
-        retry-period: 6h
+        retryperiod: 6h
         # actions are run by the filter when regexes are matched
         actions:
           # actions have a user-defined name
@@ -50,7 +50,7 @@ streams:
           unban:
             cmd:  *iptablesunban
             # if after is defined, the action will not take place immediately, but after a specified duration
-            # same format as retry-period
+            # same format as retryperiod
             after: 48h
             # let's say reaction is quitting. does it run all those pending commands which had an `after` duration set?
             # if you want reaction to run those pending commands before exiting, you can set this:

app/startup.go

@@ -1,6 +1,7 @@
 package app
 
 import (
+	"encoding/json"
 	"fmt"
 	"log"
 	"os"
@@ -8,7 +9,7 @@ import (
 	"strings"
 	"time"
 
-	"gopkg.in/yaml.v3"
+	"github.com/google/go-jsonnet"
 )
 
 func (c *Conf) setup() {
@@ -131,13 +132,21 @@ func (c *Conf) setup() {
 
 func parseConf(filename string) *Conf {
 
-	data, err := os.ReadFile(filename)
+	data, err := os.Open(filename)
 	if err != nil {
 		log.Fatalln("FATAL Failed to read configuration file:", err)
 	}
 
 	var conf Conf
-	err = yaml.Unmarshal(data, &conf)
+	if filename[len(filename)-4:] == ".yml" || filename[len(filename)-5:] == ".yaml" {
+		err = jsonnet.NewYAMLToJSONDecoder(data).Decode(&conf)
+	} else {
+		var jsondata string
+		jsondata, err = jsonnet.MakeVM().EvaluateFile(filename)
+		if err == nil {
+			err = json.Unmarshal([]byte(jsondata), &conf)
+		}
+	}
 	if err != nil {
 		log.Fatalln("FATAL Failed to parse configuration file:", err)
 	}

app/types.go

@@ -8,54 +8,54 @@ import (
 )
 
 type Conf struct {
-	Patterns map[string]*Pattern `yaml:"patterns"`
+	Patterns map[string]*Pattern `json:"patterns"`
-	Streams  map[string]*Stream  `yaml:"streams"`
+	Streams  map[string]*Stream  `json:"streams"`
 }
 
 type Pattern struct {
-	Regex  string   `yaml:"regex"`
+	Regex  string   `json:"regex"`
-	Ignore []string `yaml:"ignore"`
+	Ignore []string `json:"ignore"`
 
-	name           string `yaml:"-"`
+	name           string `json:"-"`
-	nameWithBraces string `yaml:"-"`
+	nameWithBraces string `json:"-"`
 }
 
 // Stream, Filter & Action structures must never be copied.
 // They're always referenced through pointers
 
 type Stream struct {
-	name string `yaml:"-"`
+	name string `json:"-"`
 
-	Cmd     []string           `yaml:"cmd"`
+	Cmd     []string           `json:"cmd"`
-	Filters map[string]*Filter `yaml:"filters"`
+	Filters map[string]*Filter `json:"filters"`
 }
 
 type Filter struct {
-	stream *Stream `yaml:"-"`
+	stream *Stream `json:"-"`
-	name   string  `yaml:"-"`
+	name   string  `json:"-"`
 
-	Regex         []string        `yaml:"regex"`
+	Regex         []string        `json:"regex"`
-	compiledRegex []regexp.Regexp `yaml:"-"`
+	compiledRegex []regexp.Regexp `json:"-"`
-	pattern       *Pattern        `yaml:"-"`
+	pattern       *Pattern        `json:"-"`
 
-	Retry         int           `yaml:"retry"`
+	Retry         int           `json:"retry"`
-	RetryPeriod   string        `yaml:"retry-period"`
+	RetryPeriod   string        `json:"retryperiod"`
-	retryDuration time.Duration `yaml:"-"`
+	retryDuration time.Duration `json:"-"`
 
-	Actions                map[string]*Action `yaml:"actions"`
+	Actions                map[string]*Action `json:"actions"`
 	longuestActionDuration *time.Duration
 }
 
 type Action struct {
-	filter *Filter `yaml:"-"`
+	filter *Filter `json:"-"`
-	name   string  `yaml:"-"`
+	name   string  `json:"-"`
 
-	Cmd []string `yaml:"cmd"`
+	Cmd []string `json:"cmd"`
 
-	After         string        `yaml:"after"`
+	After         string        `json:"after"`
-	afterDuration time.Duration `yaml:"-"`
+	afterDuration time.Duration `json:"-"`
 
-	OnExit bool `yaml:"onexit"`
+	OnExit bool `json:"onexit"`
 }
 
 type LogEntry struct {

config/reaction.jsonnet

@@ -0,0 +1,88 @@
+// This file is using JSONNET, a complete configuration language based on JSON
+// See https://jsonnet.org
+// JSONNET is a superset of JSON, so one can write plain JSON files if wanted.
+
+// variables defined for later use.
+local iptablesban = ['iptables', '-w', '-A', 'reaction', '1', '-s', '<ip>', '-j', 'DROP'];
+local iptablesunban = ['iptables', '-w', '-D', 'reaction', '1', '-s', '<ip>', '-j', 'DROP'];
+
+{
+  // patterns are substitued in regexes.
+  // when a filter performs an action, it replaces the found pattern
+  patterns: {
+    ip: {
+      // reaction regex syntax is defined here: https://github.com/google/re2/wiki/Syntax
+      // jsonnet's @'string' is for verbatim strings
+      regex: @'(?:(?:[0-9]{1,3}\.){3}[0-9]{1,3})|(?:[0-9a-fA-F:]{2,90})',
+      ignore: ['127.0.0.1', '::1'],
+    },
+  },
+
+  // streams are commands
+  // they're run and their ouptut is captured
+  // *example:* `tail -f /var/log/nginx/access.log`
+  // their output will be used by one or more filters
+  streams: {
+    // streams have a user-defined name
+    ssh: {
+      // note that if the command is not in environment's `PATH`
+      // its full path must be given.
+      cmd: ['journalctl', '-fu', 'sshd.service'],
+      // filters run actions when they match regexes on a stream
+      filters: {
+        // filters have a user-defined name
+        failedlogin: {
+          // reaction's regex syntax is defined here: https://github.com/google/re2/wiki/Syntax
+          regex: [
+            // <ip> is predefined in the patterns section
+            // ip's regex is inserted in the following regex
+            'authentication failure;.*rhost=<ip>',
+          ],
+          // if retry and retryperiod are defined,
+          // the actions will only take place if a same pattern is
+          // found `retry` times in a `retryperiod` interval
+          retry: 3,
+          // format is defined here: https://pkg.go.dev/time#ParseDuration
+          retryperiod: '6h',
+          // actions are run by the filter when regexes are matched
+          actions: {
+            // actions have a user-defined name
+            ban: {
+              // JSONNET substitutes the variable (defined at the beginning of the file)
+              cmd: iptablesban,
+            },
+            unban: {
+              cmd: iptablesunban,
+              // if after is defined, the action will not take place immediately, but after a specified duration
+              // same format as retryperiod
+              after: '48h',
+              // let's say reaction is quitting. does it run all those pending commands which had an `after` duration set?
+              // if you want reaction to run those pending commands before exiting, you can set this:
+              onexit: true,
+              // (defaults to false)
+              // here it is not useful because we will flush the chain containing the bans anyway
+              // (see /conf/reaction.service)
+            },
+          },
+        },
+      },
+    },
+  },
+}
+
+// persistence
+
+// tldr; when an `after` action is set in a filter, such filter acts as a 'jail',
+// which is persisted after reboots.
+
+// full;
+// when a filter is triggered, there are 2 flows:
+//
+// if none of its actions have an `after` directive set:
+// no action will be replayed.
+//
+// else (if at least one action has an `after` directive set):
+// if reaction stops while `after` actions are pending:
+// and reaction starts again while those actions would still be pending:
+// reaction executes the past actions (actions without after or with then+after < now)
+// and plans the execution of future actions (actions with then+after > now)

config/reaction.test.jsonnet

@@ -0,0 +1,30 @@
+{
+  patterns: {
+    num: {
+      regex: '[0-9]+',
+    },
+  },
+
+  streams: {
+    tailDown1: {
+      cmd: ['sh', '-c', "echo 1 2 3 4 5 1 2 3 4 5 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 1 2 3 4 | tr ' ' '\n' | while read i; do sleep 2; echo found $(($i % 10)); done"],
+      filters: {
+        findIP: {
+          regex: ['^found <num>$'],
+          retry: 3,
+          retryperiod: '30s',
+          actions: {
+            damn: {
+              cmd: ['echo', '<num>'],
+            },
+            undamn: {
+              cmd: ['echo', 'undamn', '<num>'],
+              after: '30s',
+              onexit: true,
+            },
+          },
+        },
+      },
+    },
+  },
+}

config/reaction.test.yml

@@ -15,7 +15,7 @@ streams:
         regex:
           - '^found <num>$'
         retry: 3
-        retry-period: 30s
+        retryperiod: 30s
         actions:
           damn:
             cmd: [ "echo", "<num>" ]

go.mod

@@ -3,5 +3,8 @@ module framagit.org/ppom/reaction
 go 1.19
 
 require (
-	gopkg.in/yaml.v3 v3.0.1
+	github.com/google/go-jsonnet v0.20.0
+	sigs.k8s.io/yaml v1.1.0
 )
+
+require gopkg.in/yaml.v2 v2.4.0 // indirect

go.sum

@@ -1,4 +1,10 @@
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/google/go-jsonnet v0.20.0 h1:WG4TTSARuV7bSm4PMB4ohjxe33IHT5WVTrJSU33uT4g=
+github.com/google/go-jsonnet v0.20.0/go.mod h1:VbgWF9JX7ztlv770x/TolZNGGFfiHEVx9G6ca2eUmeA=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo=
-gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
+sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=