BUGFIX: Invert TTL and data type

README.md

@@ -28,6 +28,11 @@ dramatically by making use of hot reloading. To do this, do the following:
 
 Usage
 -----
+### About data types
+* "Strings" lookup table will use GET/SET redis commands. Designed to set, get, expire, delete keys.  
+* "Streams" is designed for a "fire-and-forget" usage. Only "lookup_set_value" will work. Please note there is currently no TTL on stream, so you need to have an external process to purge your stream if you do not want it to grow indefinitely.
+
+### Usage in pipelines
 
 * Create data adapter, cache (or not), lookup table  
 * Use 'lookup_set_value(lookup_table, key, value, [ttl])' to create or update key in redis  
@@ -46,8 +51,8 @@ Be aware that only setting TTL with 'lookup_assign_ttl' of 'lookup_set_string_li
 Known bugs
 ----------
 
-Deletion via lookup_remove_string_list, lookup_clear_key or keep_duplicates=false sometimes not done.  
-
+* Deletion via lookup_remove_string_list, lookup_clear_key or keep_duplicates=false sometimes not done.  
+* lookup_add_string_list : Each character of a string list is considered an item  
 
 Getting started
 ---------------

src/main/java/in/nosd/redis/RedisLookupPluginModule.java

@@ -26,6 +26,7 @@ import org.graylog2.plugin.PluginModule;
 
 import in.nosd.redis.dataadapters.RedisLookupDataAdapter;
 import in.nosd.redis.functions.RedisLookupPluginFunction;
+import in.nosd.redis.migrations.V104_MigrateRedisType;
 
 import java.util.Collections;
 import java.util.Set;
@@ -64,8 +65,10 @@ public class RedisLookupPluginModule extends PluginModule {
          *
          * addConfigBeans();
          */
+
+        addMigration(V104_MigrateRedisType.class);
          
-         addMessageProcessorFunction(RedisLookupPluginFunction.NAME, RedisLookupPluginFunction.class);
+        addMessageProcessorFunction(RedisLookupPluginFunction.NAME, RedisLookupPluginFunction.class);
 
         installLookupDataAdapter2(RedisLookupDataAdapter.NAME, RedisLookupDataAdapter.class, 
             RedisLookupDataAdapter.Factory.class, RedisLookupDataAdapter.Config.class);

src/main/java/in/nosd/redis/dataadapters/RedisLookupDataAdapter.java

@@ -410,6 +410,7 @@ public class RedisLookupDataAdapter extends LookupDataAdapter {
                     .redisHost("127.0.0.1")
                     .redisPort(6379)
                     .redisDB(0)
+                    .redisType("strings")
                     .redisKeyTTL(-1)
                     .redisUsername("")
                     .redisPassword("")
@@ -441,6 +442,13 @@ public class RedisLookupDataAdapter extends LookupDataAdapter {
         @JsonProperty("redis_database")
         @Min(0)
         public abstract int redisDB();
+        
+        @JsonProperty("redis_type")
+        /* FIXME: This should be notEmpty, but migration crash when dbDataAdapterService.findAll() with error 
+        * "Missing required properties: redisType"
+        * so dont flag NoteEmpty and put Nullable. */
+        @Nullable
+        public abstract String redisType();
 
         @JsonProperty("redis_ttl")
         @Min(0)
@@ -488,6 +496,9 @@ public class RedisLookupDataAdapter extends LookupDataAdapter {
 
             @JsonProperty("redis_database")
             public abstract Builder redisDB(int redisDB);
+            
+            @JsonProperty("redis_type")
+            public abstract Builder redisType(String redisType);
 
             @JsonProperty("redis_ttl")
             public abstract Builder redisKeyTTL(long redisKeyTTL);

src/main/java/in/nosd/redis/migrations/V104_MigrateRedisType.java

@@ -0,0 +1,140 @@
+/*
+ * Copyright (C) 2024 johan@nosd.in
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package in.nosd.redis.migrations;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.auto.value.AutoValue;
+import com.google.common.collect.ImmutableSet;
+import org.graylog.autovalue.WithBeanGetter;
+import in.nosd.redis.dataadapters.RedisLookupDataAdapter;
+import in.nosd.redis.dataadapters.RedisLookupDataAdapter.Config;
+import org.graylog2.events.ClusterEventBus;
+import org.graylog2.lookup.db.DBDataAdapterService;
+import org.graylog2.lookup.dto.DataAdapterDto;
+import org.graylog2.lookup.events.DataAdaptersUpdated;
+import org.graylog2.migrations.Migration;
+import org.graylog2.plugin.cluster.ClusterConfigService;
+import org.graylog2.plugin.lookup.LookupDataAdapterConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import java.time.ZonedDateTime;
+import java.util.Collections;
+import java.util.Collection;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static com.google.common.base.Strings.isNullOrEmpty;
+
+// From graylog-plugin-threatintel/src/main/java/org/graylog/plugins/threatintel/migrations/V20170821100300_MigrateOTXAPIToken.java
+
+/* Manually :
+*
+* Go into mongodb instance
+* Locate "lut_data_adapters" table
+* Edit every entry for which "config.type" == "RedisLookup"
+*  Add "redis_type" key with value "strings"
+*/
+
+
+public class V104_MigrateRedisType extends Migration {
+    private static final Logger LOG = LoggerFactory.getLogger(V104_MigrateRedisType.class);
+    private static final ImmutableSet<String> REDIS_DATA_ADAPTER_NAMES = ImmutableSet.of("redis-lookup");
+
+    private final ClusterConfigService clusterConfigService;
+    private final DBDataAdapterService dbDataAdapterService;
+    private final ClusterEventBus clusterBus;
+
+    @Inject
+    public V104_MigrateRedisType(ClusterConfigService clusterConfigService,
+                                 DBDataAdapterService dbDataAdapterService,
+                                 ClusterEventBus clusterBus) {
+        this.clusterConfigService = clusterConfigService;
+        this.dbDataAdapterService = dbDataAdapterService;
+        this.clusterBus = clusterBus;
+    }
+
+    @Override
+    public ZonedDateTime createdAt() {
+        return ZonedDateTime.parse("2024-03-03T15:42:42Z");
+    }
+
+    @Override
+    public void upgrade() {
+        Config redisConf;
+
+        // List every instance of lookup DB adapter, and find instance with Config.type == "RedisLookup" (RedisLookupDataAdapter.NAME)
+        Collection<DataAdapterDto> DataAdapters = dbDataAdapterService.findAll(); 
+        for (DataAdapterDto dataAdapterDto : DataAdapters) {
+            if (dataAdapterDto.config().type().equals(RedisLookupDataAdapter.NAME)) {
+                redisConf = (Config) dataAdapterDto.config();
+                
+                final Config newConf = Config.builder()
+                                            .type(redisConf.type())
+                                            .redisHost(redisConf.redisHost())
+                                            .redisPort(redisConf.redisPort())
+                                            .redisDB(redisConf.redisDB())
+                                            .redisType("strings")
+                                            .redisKeyTTL(redisConf.redisKeyTTL())
+                                            .redisUsername(redisConf.redisUsername())
+                                            .redisPassword(redisConf.redisPassword())
+                                            .build();
+                
+                final DataAdapterDto newDto = DataAdapterDto.builder()
+                    .id(dataAdapterDto.id())
+                    .config(newConf)
+                    .title(dataAdapterDto.title())
+                    .description(dataAdapterDto.description())
+                    .name(dataAdapterDto.name())
+                    .build();
+
+                final DataAdapterDto saved = dbDataAdapterService.save(newDto);
+                clusterBus.post(DataAdaptersUpdated.create(saved.id()));
+                LOG.debug("Redis data adapter <{}> migrated: data_type added", dataAdapterDto.name());
+            }
+        }
+    }
+
+    @JsonAutoDetect
+    @AutoValue
+    @WithBeanGetter
+    public static abstract class MigrationCompleted {
+        @JsonProperty("converted_redis_type")
+        public abstract boolean convertedRedisType();
+
+        @JsonProperty("data_adapter_ids")
+        public abstract Set<String> dataAdapterIds();
+
+        @JsonCreator
+        public static MigrationCompleted create(@JsonProperty("data_adapter_ids") final Set<String> dataAdapterIds,
+                                                @JsonProperty("converted_redis_type") final boolean convertedRedisType) {
+            return new AutoValue_V104_MigrateRedisType_MigrationCompleted(convertedRedisType, dataAdapterIds);
+        }
+
+        public static MigrationCompleted convertedType(@JsonProperty("data_adapter_ids") final Set<String> dataAdapterIds) {
+            return create(dataAdapterIds, true);
+        }
+
+        public static MigrationCompleted notConvertedType() {
+            return create(Collections.emptySet(), false);
+        }
+    }
+}
+ 

src/web/dataadapters/RedisLookupAdapterFieldSet.jsx

@@ -16,13 +16,23 @@
  */
 import React, { useCallback, useEffect, useRef, useState } from 'react';
 import PropTypes from 'prop-types';
-//import { Button } from 'components/graylog';
+
+//import ObjectUtils from 'util/ObjectUtils';
 import { Input } from 'components/bootstrap';
+// "Missing or invalid plugin" in "dataAdapter create/Data adapter type" list when imported
+//import { Select } from './components/common';
+//import { Select } from '../../../../graylog2-server/graylog2-web-interface/src/components/common';
 
 class RedisLookupAdapterFieldSet extends React.Component {
   static propTypes = {
     config: PropTypes.shape({
       redis_host: PropTypes.string.isRequired,
+      redis_port: PropTypes.number.isRequired,
+      redis_database: PropTypes.number.isRequired,
+      redis_type: PropTypes.string.isRequired,
+      redis_ttl: PropTypes.number.isRequired,
+      redis_username: PropTypes.string,
+      redis_password: PropTypes.string,
     }).isRequired,
     updateConfig: PropTypes.func.isRequired,
     handleFormEvent: PropTypes.func.isRequired,
@@ -30,14 +40,14 @@ class RedisLookupAdapterFieldSet extends React.Component {
     validationMessage: PropTypes.func.isRequired,
   };
   
-  handleSelect = (fieldName) => {
-    return (selectedIndicator) => {
-      const config = lodash.cloneDeep(this.props.config);
-      config[fieldName] = selectedIndicator;
-      this.props.updateConfig(config);
-    };
+/*  _onRedisTypeSelect = (type) => {
+    const { config, updateConfig } = this.props;
+    const newConfig = ObjectUtils.clone(config);
+    
+    newConfig.redis_type = type;
+    updateConfig(newConfig);
   };
-
+*/
   render() {
     const { config } = this.props;
  
@@ -76,6 +86,18 @@ class RedisLookupAdapterFieldSet extends React.Component {
         value={config.redis_database}
         labelClassName="col-sm-3"
         wrapperClassName="col-sm-9" />
+      <Input type="text"
+        id="redis_type"
+        name="redis_type"
+        label="Redis data type"
+        required
+        onChange={this.props.handleFormEvent}
+        help={this.props.validationMessage('redis_type', 'Redis data type used for lookups. Should be one of "strings", "streams".')}
+        bsStyle={this.props.validationState('redis_type')}
+        value={config.redis_type}
+        labelClassName="col-sm-3"
+        wrapperClassName="col-sm-9">
+      </Input>
       <Input type="text"
         id="redis_ttl"
         name="redis_ttl"

src/web/dataadapters/RedisLookupAdapterSummary.jsx

@@ -40,6 +40,8 @@ class RedisLookupAdapterSummary extends React.Component {
       <dd>{config.redis_database}</dd>
       <dt>Redis key TTL</dt>
       <dd>{config.redis_ttl || 'n/a'}</dd>
+      <dt>Redis data type</dt>
+      <dd>{config.redis_type || 'n/a'}</dd>
       <dt>Redis username</dt>
       <dd>{config.redis_username || 'n/a'}</dd>
       <dt>Redis password</dt>