v1.0.3: Support string list, with TTL

README.md

@@ -1,6 +1,6 @@
 # RedisLookupPlugin Plugin for Graylog
 
-Plugin to add Redis Data Adapter in read/write to graylog so you can store and retrieve key/values from pipelines  
+Plugin to add Redis Data Adapter in read/write to graylog so you can store and retrieve key/values and lists of string values from pipelines  
 Support Redis authentication (with password and username/password)  
 
 **Required Graylog version:** 5.0 and later
@@ -8,7 +8,7 @@ Support Redis authentication (with password and username/password)
 Installation
 ------------
 
-[Download the plugin](https://git.nosd.in/yo/graylog-redis-lookup-plugin/releases/download/v1.0.0/graylog-plugin-redis-lookup-1.0.0.jar)
+[Download the plugin](https://git.nosd.in/yo/graylog-redis-lookup-plugin/releases/download/v1.0.3/graylog-plugin-redis-lookup-1.0.3.jar)
 and place the `.jar` file in your Graylog plugin directory. The plugin directory
 is the `plugins/` folder relative from your `graylog-server` directory by default
 and can be configured in your `graylog.conf` file.
@@ -35,8 +35,18 @@ Usage
 * Use 'lookup_clear_key(lookup_table, key)' to remove key  
 * Use 'lookup_has_value(lookup_table, key)' to test key existence  
 * Use 'lookup_assign_ttl(lookup_table, key, ttl)' to change TTL of existing key  
+* Use 'lookup_set_string_list(lookup_table, key, value, [ttl])' to create a list named "key"  
+* Use 'lookup_add_string_list(lookup_table, key, value, [keep_duplicates])' to add value list to existing list  
+* Use 'lookup_remove_string_list(lookup_table, key, value) to remove a string from list "key"  
 
-By default keys will be created in Redis with the default TTL defined at data adapter creation time  
+By default single value keys will be created in Redis with the default TTL defined at data adapter creation time  
+
+Be aware that only setting TTL with 'lookup_assign_ttl' of 'lookup_set_string_list' alter TTL value in Redis ; so a list created with a TTL of 3600 will expire in 3600 seconds, even if it was updated with 'lookup_add_string_list' some seconds before expiration.  
+
+Known bugs
+----------
+
+Deletion via lookup_remove_string_list, lookup_clear_key or keep_duplicates=false sometimes not done.  
 
 
 Getting started

dependency-reduced-pom.xml

@@ -10,7 +10,7 @@
   <groupId>in.nosd.redis</groupId>
   <artifactId>graylog-plugin-redis-lookup</artifactId>
   <name>${project.artifactId}</name>
-  <version>1.0.1</version>
+  <version>1.0.3</version>
   <description>Graylog ${project.artifactId} plugin.</description>
   <url>https://www.graylog.org</url>
   <developers>

pom.xml

@@ -31,7 +31,7 @@
 
     <groupId>in.nosd.redis</groupId>
     <artifactId>graylog-plugin-redis-lookup</artifactId>
-    <version>1.0.1</version>
+    <version>1.0.3</version>
     <packaging>jar</packaging>
 
     <name>${project.artifactId}</name>

src/main/java/in/nosd/redis/dataadapters/RedisLookupDataAdapter.java

@@ -73,7 +73,18 @@ public class RedisLookupDataAdapter extends LookupDataAdapter {
     private final Meter redisGetRequestErrors;
     private final Timer redisSetRequestTimer;
     private final Meter redisSetRequestErrors;
-
+    private final Timer redisDelRequestTimer;
+    private final Meter redisDelRequestErrors;
+    private final Timer redisAssignTtlRequestTimer;
+    private final Meter redisAssignTtlRequestErrors;
+    private final Timer redisAddStringListRequestTimer;
+    private final Meter redisAddStringListRequestErrors;
+    private final Timer redisSetStringListRequestTimer;
+    private final Meter redisSetStringListRequestErrors;
+    private final Timer redisSetStringListWithTtlRequestTimer;
+    private final Meter redisSetStringListWithTtlRequestErrors;
+    private final Timer redisRemoveStringListRequestTimer;
+    private final Meter redisRemoveStringListRequestErrors;
 
     @Inject
     public RedisLookupDataAdapter(@Assisted("dto") DataAdapterDto dto,
@@ -98,6 +109,18 @@ public class RedisLookupDataAdapter extends LookupDataAdapter {
         this.redisGetRequestErrors = metricRegistry.meter(MetricRegistry.name(getClass(), "redisGetRequestErrors"));
         this.redisSetRequestTimer = metricRegistry.timer(MetricRegistry.name(getClass(), "redisSetRequestTime"));
         this.redisSetRequestErrors = metricRegistry.meter(MetricRegistry.name(getClass(), "redisSetRequestErrors"));
+        this.redisDelRequestTimer = metricRegistry.timer(MetricRegistry.name(getClass(), "redisDelRequestTime"));
+        this.redisDelRequestErrors = metricRegistry.meter(MetricRegistry.name(getClass(), "redisDelRequestErrors"));
+        this.redisAssignTtlRequestTimer = metricRegistry.timer(MetricRegistry.name(getClass(), "redisAssignTtlRequestTime"));
+        this.redisAssignTtlRequestErrors = metricRegistry.meter(MetricRegistry.name(getClass(), "redisAssignTtlRequestErrors"));
+        this.redisAddStringListRequestTimer = metricRegistry.timer(MetricRegistry.name(getClass(), "redisAddStringListRequestTime"));
+        this.redisAddStringListRequestErrors = metricRegistry.meter(MetricRegistry.name(getClass(), "redisAddStringListRequestErrors"));
+        this.redisSetStringListRequestTimer = metricRegistry.timer(MetricRegistry.name(getClass(), "redisSetStringListRequestTime"));
+        this.redisSetStringListRequestErrors = metricRegistry.meter(MetricRegistry.name(getClass(), "redisSetStringListRequestErrors"));
+        this.redisSetStringListWithTtlRequestTimer = metricRegistry.timer(MetricRegistry.name(getClass(), "redisSetStringListWithTtlRequestTime"));
+        this.redisSetStringListWithTtlRequestErrors = metricRegistry.meter(MetricRegistry.name(getClass(), "redisSetStringListWithTtlRequestErrors"));
+        this.redisRemoveStringListRequestTimer = metricRegistry.timer(MetricRegistry.name(getClass(), "redisRemoveStringListRequestTime"));
+        this.redisRemoveStringListRequestErrors = metricRegistry.meter(MetricRegistry.name(getClass(), "redisRemoveStringListRequestErrors"));
     }
 
     @Override
@@ -124,6 +147,11 @@ public class RedisLookupDataAdapter extends LookupDataAdapter {
         cachePurge.purgeAll();
     }
 
+    private String getSingleValue(String key) {
+        final String value = this.commands.get(key);
+        return value;
+    }
+
     @Override
     protected LookupResult doGet(Object key) {
         final Timer.Context time = redisGetRequestTimer.time();
@@ -133,15 +161,22 @@ public class RedisLookupDataAdapter extends LookupDataAdapter {
             return getEmptyResult();
         }
         try {
-            final String value = this.commands.get(trimmedKey);
-            if (value == null) {
-                LOG.debug("Redis GET request for key <{}> returned null, key do not exists.", trimmedKey);
-                redisGetRequestErrors.mark();
-                return LookupResult.empty();
+            // Get item type and existence
+            final String type = this.commands.type(trimmedKey);
+            switch(type) {
+                case "none":
+                    LOG.debug("Redis TYPE request for key <{}> returned null, key do not exists.", trimmedKey);
+                    redisGetRequestErrors.mark();
+                    return LookupResult.empty();
+                case "list":
+                    final List<String> result = this.commands.lrange(trimmedKey, 0, -1);
+                    return LookupResult.withoutTTL().stringListValue(result).build();
+                default:
+                    final String value = getSingleValue(trimmedKey);
+                    return LookupResult.single(value);
             }
-            return LookupResult.single(value);
         } catch (Exception e) {
-            LOG.error("Redis GET request error for key <{}>", trimmedKey, e);
+            LOG.error("Exception: Redis GET request error for key <{}>", trimmedKey, e);
             redisGetRequestErrors.mark();
             return LookupResult.empty();
         } finally {
@@ -164,15 +199,20 @@ public class RedisLookupDataAdapter extends LookupDataAdapter {
         final Timer.Context time = redisSetRequestTimer.time();
         final String trimmedKey = StringUtils.trimToNull(key.toString());
         try {
-            final String result = this.commands.setex(trimmedKey, ttlSec, value.toString());
+            final String result;
+            if (ttlSec > 0 ) {
+                result = this.commands.setex(trimmedKey, ttlSec, value.toString());
+            } else {
+                result = this.commands.set(trimmedKey, value.toString());
+            }
             if (!result.equals("OK")) {
-                LOG.warn("Redis SETEX key <{}> to value <{}> with TTL <{}> returned {}", key, value, ttlSec, result);
+                LOG.warn("Redis SET(EX) key <{}> to value <{}> with TTL <{}> returned {}", key, value, ttlSec, result);
                 redisSetRequestErrors.mark();
                 return LookupResult.empty();
             }
             return LookupResult.single(value.toString());
         } catch (Exception e) {
-            LOG.error("Redis SETEX key <{}> to value <{}> with TTL <{}> returned an exception: {}", key, value, ttlSec, e);
+            LOG.error("Exception: Redis SET(EX) key <{}> to value <{}> with TTL <{}> returned {}", key, value, ttlSec, e);
             redisSetRequestErrors.mark();
             return LookupResult.withError();
         } finally {
@@ -182,40 +222,169 @@ public class RedisLookupDataAdapter extends LookupDataAdapter {
 
     @Override
     public void clearKey(Object key) {
-        final Timer.Context time = redisSetRequestTimer.time();
+        final Timer.Context time = redisDelRequestTimer.time();
         final String trimmedKey = StringUtils.trimToNull(key.toString());
         try {
-            final Long result = this.commands.del(key.toString());
+            final Long result = this.commands.del(trimmedKey);
             if (result != 1) {
-                LOG.debug("Redis DEL key <{}> returned {}", key, result);
-                redisSetRequestErrors.mark();
+                LOG.debug("Redis DEL key <{}> returned {}", trimmedKey, result);
+                redisDelRequestErrors.mark();
             }
             return;
         } catch (Exception e) {
-            LOG.error("Redis DEL key <{}> returned {}", key, e);
-            redisSetRequestErrors.mark();
+            LOG.error("Exception: Redis DEL key <{}> returned {}", trimmedKey, e);
+            redisDelRequestErrors.mark();
             return;
         } finally {
             time.stop();
         }
     }
 
-    public LookupResult assignTtl(Object key, Long ttlSec) {
-        final Timer.Context time = redisSetRequestTimer.time();
-        final String trimmedKey = StringUtils.trimToNull(key.toString());
+    private LookupResult setExpire(String key, Long ttl) {
         try {
-            final Boolean result = this.commands.expire(trimmedKey, ttlSec);
-            if (!result) {
-                LOG.warn("Redis EXPIRE key <{}> to <{}> returned {}", key, ttlSec, result);
-                redisSetRequestErrors.mark();
+            if (!this.commands.expire(key, ttl)) {
+                LOG.warn("Redis EXPIRE key <{}> to <{}> returned false (key does not exist or the timeout could not be set)", key, ttl);
                 return LookupResult.withError();
             }
-            final String value = this.commands.get(trimmedKey);
-            return LookupResult.single(value.toString());
+            return LookupResult.single(this.commands.get(key).toString());
+        }
+        catch (Exception e) {
+            // lettuce 6.3.0 returns "WRONGTYPE Operation against a key holding the wrong kind of value" when EXPIRE on a list key, but do the job.
+            if (e.getMessage().startsWith("WRONGTYPE Operation against a key holding the wrong kind of value")) {
+            } else {
+                LOG.error("Exception: Redis EXPIRE key <{}> to <{}> returned {}", key, ttl, e);
+                return LookupResult.withError(e.toString());
+            }
+        }
+        return LookupResult.single(this.commands.get(key).toString());
+    }
+
+    private LookupResult setPersist(String key) {
+        try {
+            if (!this.commands.persist(key)) {
+                LOG.debug("Redis PERSIST key <{}> returned false (key does not exist or does not have an associated timeout)", key);
+                return LookupResult.withError();
+            }
+            return LookupResult.single(this.commands.get(key).toString());
         } catch (Exception e) {
-            LOG.error("Redis EXPIRE key <{}> to <{}> returned {}", key, ttlSec, e);
-            redisSetRequestErrors.mark();
-            return LookupResult.withError();
+            LOG.error("Exception: Redis PERSIST key <{}> returned {}", key, e);
+            return LookupResult.withError(e.toString());
+        }
+    }
+
+    // TTL -1 = never expire
+    public LookupResult assignTtl(Object key, Long ttlSec) {
+        final Timer.Context time = redisAssignTtlRequestTimer.time();
+        final String trimmedKey = StringUtils.trimToNull(key.toString());
+        try {
+            if (ttlSec > 0) {
+                return setExpire(trimmedKey, ttlSec);
+            } else {
+                return setPersist(trimmedKey);
+            }
+        } catch (Exception e) {
+            // lettuce 6.3.0 returns "WRONGTYPE Operation against a key holding the wrong kind of value" when TTL on a list key, but do the job.
+            if (e.getMessage().startsWith("WRONGTYPE Operation against a key holding the wrong kind of value")) {
+            } else {
+                LOG.error("Exception: assignTtl <{}> to key <{}> returned {}", ttlSec, trimmedKey, e);
+                redisAssignTtlRequestErrors.mark();
+                return LookupResult.withError();
+            }
+        } finally {
+            time.stop();
+        }
+        return LookupResult.single(this.commands.get(trimmedKey));
+    }
+
+    @Override
+    public LookupResult addStringList(Object key, List<String> listValue, boolean keepDuplicates) {
+        final Timer.Context time = redisAddStringListRequestTimer.time();
+        final String trimmedKey = StringUtils.trimToNull(key.toString());
+        if (trimmedKey == null) {
+            LOG.debug("A blank key was supplied");
+            return getEmptyResult();
+        }
+        try {
+            if (!keepDuplicates) {
+                removeStringList(trimmedKey, listValue);
+            }
+            final Long len = this.commands.rpush(trimmedKey, listValue.toArray(new String[0]));
+            if (len > 0) {
+                return LookupResult.withoutTTL().stringListValue(this.commands.lrange(trimmedKey, 0, -1)).build();
+            }
+            return LookupResult.empty();
+        } catch (Exception e) {
+            LOG.error("Exception: Redis RPUSH request error for key <{}>: <{}>", trimmedKey, e);
+            redisAddStringListRequestErrors.mark();
+            return LookupResult.empty();
+        } finally {
+            time.stop();
+        }
+    }
+
+    @Override
+    public LookupResult setStringList(Object key, List<String> listValue) {
+        final Timer.Context time = redisSetStringListRequestTimer.time();
+        final String trimmedKey = StringUtils.trimToNull(key.toString());
+        if (trimmedKey == null) {
+            LOG.debug("A blank key was supplied");
+            return getEmptyResult();
+        }
+        try {
+            // We need to replace list, so we delete it first
+            this.commands.ltrim(trimmedKey, 1, 0);
+            final Long len = this.commands.rpush(trimmedKey, listValue.toArray(new String[0]));
+            if (len > 0) {
+                return LookupResult.withoutTTL().stringListValue(this.commands.lrange(trimmedKey, 0, -1)).build();
+            }
+            return LookupResult.empty();
+        } catch (Exception e) {
+            LOG.error("Exception: Redis RPUSH request error for key <{}>: <{}>", trimmedKey, e);
+            redisSetStringListRequestErrors.mark();
+            return LookupResult.empty();
+        } finally {
+            time.stop();
+        }
+    }
+
+    public LookupResult setStringListWithTtl(Object key, List<String> listValue, Long ttlSec) {
+        final Timer.Context time = redisSetStringListWithTtlRequestTimer.time();
+        try {
+            setStringList(key, listValue);
+            return(assignTtl(key, ttlSec));
+        } catch (Exception e) {
+            // This exception comes from assignTtl
+            if (e.getMessage().startsWith("WRONGTYPE Operation against a key holding the wrong kind of value")) {
+            } else {
+                LOG.error("Exception: Redis RPUSH request error for key <{}>: <{}>", key, e);
+                redisSetStringListWithTtlRequestErrors.mark();
+                return LookupResult.empty();
+            }
+        } finally {
+            time.stop();
+        }
+        return LookupResult.withoutTTL().stringListValue(this.commands.lrange(StringUtils.trimToNull(key.toString()), 0, -1)).build();
+    }
+
+    @Override
+    public LookupResult removeStringList(Object key, List<String> listValue) {
+        final Timer.Context time = redisRemoveStringListRequestTimer.time();
+        final String trimmedKey = StringUtils.trimToNull(key.toString());
+        if (trimmedKey == null) {
+            LOG.debug("A blank key was supplied");
+            return getEmptyResult();
+        }
+        try {
+            long removed = 0;
+            for (String value : listValue) {
+                removed += this.commands.lrem(trimmedKey, 0, value);
+            }
+            LOG.debug("Redis LREM for key <{}> and value <{}> deleted <{}> items", trimmedKey, listValue, removed);
+            return LookupResult.withoutTTL().stringListValue(this.commands.lrange(trimmedKey, 0, -1)).build();
+        } catch (Exception e) {
+            LOG.error("Exception: Redis LREM request error for key <{}> and value <{}>: <{}>", trimmedKey, listValue, e);
+            redisRemoveStringListRequestErrors.mark();
+            return LookupResult.empty();
         } finally {
             time.stop();
         }
@@ -241,7 +410,7 @@ public class RedisLookupDataAdapter extends LookupDataAdapter {
                     .redisHost("127.0.0.1")
                     .redisPort(6379)
                     .redisDB(0)
-                    .redisKeyTTL(86400)
+                    .redisKeyTTL(-1)
                     .redisUsername("")
                     .redisPassword("")
                     .build();

src/main/java/in/nosd/redis/functions/RedisLookupPluginFunction.java

@@ -76,7 +76,7 @@ public class RedisLookupPluginFunction extends LookupTableFunction<GenericLookup
     public FunctionDescriptor<GenericLookupResult> descriptor() {
         return FunctionDescriptor.<GenericLookupResult>builder()
                 .name(NAME)
-                .description("Match a key into Redis instance and return value")
+                .description("Match a key into Redis instance and return value. Do not use, prefer standard 'lookup_*' functions")
                 .params(keyParam)
                 .returnType(GenericLookupResult.class)
                 .build();

src/web/dataadapters/RedisLookupAdapterFieldSet.jsx

@@ -82,7 +82,7 @@ class RedisLookupAdapterFieldSet extends React.Component {
         label="Redis key TTL"
         required
         onChange={this.props.handleFormEvent}
-        help={this.props.validationMessage('redis_ttl', 'Redis key TTL in seconds')}
+        help={this.props.validationMessage('redis_ttl', 'Redis key TTL in seconds. Set -1 to not expire keys')}
         bsStyle={this.props.validationState('redis_ttl')}
         value={config.redis_ttl}
         labelClassName="col-sm-3"