outputs: examples and heavy load tests

config/example_streamed_output.yml

@@ -0,0 +1,59 @@
+---
+concurrency: 0
+
+# patterns are substitued in regexes.
+# when a filter performs an action, it replaces the found pattern
+patterns:
+  ip:
+    # reaction regex syntax is defined here: https://github.com/google/re2/wiki/Syntax
+    # simple version: regex: '(?:(?:[0-9]{1,3}\.){3}[0-9]{1,3})|(?:[0-9a-fA-F:]{2,90})'
+    regex: '(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(?:\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}|(?:(?:[0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(?::[0-9a-fA-F]{1,4}){1,2}|(?:[0-9a-fA-F]{1,4}:){1,4}(?::[0-9a-fA-F]{1,4}){1,3}|(?:[0-9a-fA-F]{1,4}:){1,3}(?::[0-9a-fA-F]{1,4}){1,4}|(?:[0-9a-fA-F]{1,4}:){1,2}(?::[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:(?:(?::[0-9a-fA-F]{1,4}){1,6})|:(?:(?::[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(?::[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(?:ffff(?::0{1,4}){0,1}:){0,1}(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])|(?:[0-9a-fA-F]{1,4}:){1,4}:(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]))'
+    ignore:
+      - 127.0.0.1
+      - ::1
+    # Patterns can be ignored based on regexes, it will try to match the whole string detected by the pattern
+    # ignoreregex:
+    #   - '10\.0\.[0-9]{1,3}\.[0-9]{1,3}'
+  login:
+    regex: '[a-zA-Z0-9_\-\.]*'
+    
+  method:
+    regex: '.*'
+    
+  port:
+    regex: '[0-9]{1,5}'
+
+# Outputs are commands returning stdin you can use in write actions.
+# This can ben used to get a persistent connection to p.e. a KV database you will write into,
+#  eliminating the overhead of executing a process each time action is trigged.
+outputs:
+  redis:
+    start: ['redis-cli', '-h', 'redis.example.org', '-a', 'mypasswordoncmdlinedontdothis']
+#  tee:
+#    start: ['tee', 'output.log']
+
+    
+# streams are commands
+# they are run and their ouptut is captured
+# *example:* `tail -f /var/log/nginx/access.log`
+# their output will be used by one or more filters
+streams:
+  # streams have a user-defined name
+  ssh:
+    # note that if the command is not in environment's `PATH`
+    # its full path must be given.
+    cmd: ['tail', '-f', '/var/log/auth.log']
+    # filters run actions when they match regexes on a stream
+    filters:
+      # filters have a user-defined name
+      acceptedlogin:
+        # reaction's regex syntax is defined here: https://github.com/google/re2/wiki/Syntax
+        regex:
+          - 'Accepted <method> for <login> from <ip> port <port>'
+        # actions are run by the filter when regexes are matched
+        actions:
+          # actions have a user-defined name
+          store2redis:
+            write:
+              output: redis
+              text: ['XADD', 'logins', '*', 'username', '<login>', 'method', '<method>', 'ip', '<ip>', 'port', '<port>']

config/heavy-load_cmd_to_redis.yml

@@ -0,0 +1,50 @@
+---
+patterns:
+  num:
+    regex: '[0-9]+'
+  idx:
+    regex: '[0-9]+'
+  ip:
+    regex: '(?:(?:[0-9]{1,3}\.){3}[0-9]{1,3})|(?:[0-9a-fA-F:]{2,90})'
+    ignore:
+      - 1.0.0.1
+
+concurrency: 0
+
+streams:
+  tailDown1:
+    cmd: [ 'sh', '-c', 'sleep 2; seq 100010 | while read i; do echo found $(($i % 100)) for test 1; done' ]
+    filters:
+      findIP:
+        regex:
+          - '^found <num> for test <idx>$'
+        actions:
+          store2redis:
+            cmd: ['redis-cli', '-h', 'redis.example.org', '-a', 'mypasswordoncmdlinedontdothis', 'XADD', 'teststream', '*', 'found', '<num>', 'test', '<idx>']
+  tailDown2:
+    cmd: [ 'sh', '-c', 'sleep 2; seq 100010 | while read i; do echo prout $(($i % 100)) for test 2; done' ]
+    filters:
+      findIP:
+        regex:
+          - '^prout <num> for test <idx>$'
+        actions:
+          store2redis:
+            cmd: ['redis-cli', '-h', 'redis.example.org', '-a', 'mypasswordoncmdlinedontdothis', 'XADD', 'teststream', '*', 'found', '<num>', 'test', '<idx>']
+  tailDown3:
+    cmd: [ 'sh', '-c', 'sleep 2; seq 100010 | while read i; do echo nanana $(($i % 100)) for test 3; done' ]
+    filters:
+      findIP:
+        regex:
+          - '^nanana <num> for test <idx>$'
+        actions:
+          store2redis:
+            cmd: ['redis-cli', '-h', 'redis.example.org', '-a', 'mypasswordoncmdlinedontdothis', 'XADD', 'teststream', '*', 'found', '<num>', 'test', '<idx>']
+  tailDown4:
+    cmd: [ 'sh', '-c', 'sleep 2; seq 100010 | while read i; do echo nanana $(($i % 100)) for test 4; done' ]
+    filters:
+      findIP:
+        regex:
+          - '^nomatch <num> for test <idx>$'
+        actions:
+          store2redis:
+            cmd: ['redis-cli', '-h', 'redis.example.org', '-a', 'mypasswordoncmdlinedontdothis', 'XADD', 'teststream', '*', 'found', '<num>', 'test', '<idx>']

config/heavy-load_write_to_redis.yml

@@ -0,0 +1,62 @@
+---
+patterns:
+  num:
+    regex: '[0-9]+'
+  idx:
+    regex: '[0-9]+'
+  ip:
+    regex: '(?:(?:[0-9]{1,3}\.){3}[0-9]{1,3})|(?:[0-9a-fA-F:]{2,90})'
+    ignore:
+      - 1.0.0.1
+
+concurrency: 0
+
+outputs:
+  redis:
+    start: ['redis-cli', '-h', 'redis.example.org', '-a', 'mypasswordoncmdlinedontdothis']
+
+streams:
+  tailDown1:
+    cmd: [ 'sh', '-c', 'seq 100010 | while read i; do echo found $(($i % 100)) for test 1; done' ]
+    filters:
+      findIP:
+        regex:
+          - '^found <num> for test <idx>$'
+        actions:
+          store2redis:
+            write:
+              output: redis
+              text: ['XADD', 'teststream', '*', 'found', '<num>', 'test', '<idx>']
+  tailDown2:
+    cmd: [ 'sh', '-c', 'seq 100010 | while read i; do echo prout $(($i % 100)) for test 2; done' ]
+    filters:
+      findIP:
+        regex:
+          - '^prout <num> for test <idx>$'
+        actions:
+          store2redis:
+            write:
+              output: redis
+              text: ['XADD', 'teststream', '*', 'prout', '<num>', 'test', '<idx>']
+  tailDown3:
+    cmd: [ 'sh', '-c', 'seq 100010 | while read i; do echo nanana $(($i % 100)) for test 3; done' ]
+    filters:
+      findIP:
+        regex:
+          - '^nanana <num> for test <idx>$'
+        actions:
+          store2redis:
+            write:
+              output: redis
+              text: ['XADD', 'teststream', '*', 'nanana', '<num>', 'test', '<idx>']
+  tailDown4:
+    cmd: [ 'sh', '-c', 'seq 100010 | while read i; do echo nanana $(($i % 100)) for test 4; done' ]
+    filters:
+      findIP:
+        regex:
+          - '^nomatch <num> for test <idx>$'
+        actions:
+          store2redis:
+            write:
+              output: redis
+              text: ['XADD', 'teststream', '*', 'nomatch', '<num>', 'test', '<idx>']