Compare commits

..

6 Commits

18 changed files with 4150 additions and 3332 deletions

10
.mvn/jvm.config Normal file
View File

@ -0,0 +1,10 @@
--add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
--add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
--add-exports jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED
--add-exports jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED
--add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
--add-exports jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED
--add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
--add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
--add-opens jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED
--add-opens jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED

View File

@ -28,6 +28,11 @@ dramatically by making use of hot reloading. To do this, do the following:
Usage
-----
### About data types
* "Strings" lookup table will use GET/SET redis commands. Designed to set, get, expire, delete keys.
* "Streams" is designed for a "fire-and-forget" usage. Only "lookup_set_value" will work. Please note there is currently no TTL on stream, so you need to have an external process to purge your stream if you do not want it to grow indefinitely.
### Usage in pipelines
* Create data adapter, cache (or not), lookup table
* Use 'lookup_set_value(lookup_table, key, value, [ttl])' to create or update key in redis
@ -46,8 +51,8 @@ Be aware that only setting TTL with 'lookup_assign_ttl' of 'lookup_set_string_li
Known bugs
----------
Deletion via lookup_remove_string_list, lookup_clear_key or keep_duplicates=false sometimes not done.
* Deletion via lookup_remove_string_list, lookup_clear_key or keep_duplicates=false sometimes not done.
* lookup_add_string_list : Each character of a string list is considered an item
Getting started
---------------

File diff suppressed because it is too large Load Diff

180
pom.xml
View File

@ -25,13 +25,13 @@
<parent>
<groupId>org.graylog.plugins</groupId>
<artifactId>graylog-plugin-web-parent</artifactId>
<version>6.1.0</version>
<version>5.0.0</version>
<relativePath>../graylog2-server/graylog-plugin-parent/graylog-plugin-web-parent</relativePath>
</parent>
<groupId>in.nosd.redis</groupId>
<artifactId>graylog-plugin-redis-lookup</artifactId>
<version>1.0.4</version>
<version>1.0.3</version>
<packaging>jar</packaging>
<name>${project.artifactId}</name>
@ -54,65 +54,56 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.release>11</maven.compiler.release>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
<maven.compiler.useIncrementalCompilation>false</maven.compiler.useIncrementalCompilation>
<maven.compiler.source>17</maven.compiler.source>
<maven.compiler.target>17</maven.compiler.target>
<!-- Plugins will not be deployed by default - set to `false` if you actually want to deploy it -->
<maven.deploy.skip>true</maven.deploy.skip>
<graylog.plugin-dir>/usr/local/share/graylog/plugin</graylog.plugin-dir>
<graylog.version>6.0.0</graylog.version>
<guice.version>7.0.0</guice.version>
<auto-value.version>1.10.4</auto-value.version>
<auto-value-javabean.version>2.5.2</auto-value-javabean.version>
<jackson.version>2.17.1</jackson.version>
<validation-api.version>3.0.2</validation-api.version>
<joda-time.version>2.12.7</joda-time.version>
<commons-lang3.version>3.14.0</commons-lang3.version>
<metrics.version>4.2.25</metrics.version>
<javax.annotation-api.version>1.3.2</javax.annotation-api.version>
<license-maven.version>4.3</license-maven.version>
<!-- Plugin versions -->
<license-maven.version>4.3</license-maven.version>
<download-maven-plugin.version>1.6.8.1</download-maven-plugin.version>
<graylog.version>${project.parent.version}</graylog.version>
<graylog.plugin-dir>/usr/share/graylog-server/plugin</graylog.plugin-dir>
</properties>
<!-- <distributionManagement>
<snapshotRepository>
<id>sonatype-nexus-snapshots</id>
<name>Sonatype Nexus Snapshots</name>
<url>https://oss.sonatype.org/content/repositories/snapshots</url>
</snapshotRepository>
<repository>
<id>sonatype-nexus-staging</id>
<name>Nexus Release Repository</name>
<url>https://oss.sonatype.org/service/local/staging/deploy/maven2/</url>
</repository>
</distributionManagement>
<repositories>
--> <!-- to make our snapshot releases work with Travis et al -->
<!-- <repository>
<id>sonatype-nexus-snapshots</id>
<name>Sonatype Nexus Snapshots</name>
<url>https://oss.sonatype.org/content/repositories/snapshots</url>
<releases>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
</repository>
<repository>
<id>sonatype-nexus-releases</id>
<name>Sonatype Nexus Releases</name>
<url>https://oss.sonatype.org/content/repositories/releases</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
</repositories>
-->
<dependencies>
<!-- com.codahale.metrics -->
<dependency>
<groupId>io.dropwizard.metrics</groupId>
<artifactId>metrics-core</artifactId>
<version>${metrics.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>jakarta.validation</groupId>
<artifactId>jakarta.validation-api</artifactId>
<version>${validation-api.version}</version>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>${joda-time.version}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>${commons-lang3.version}</version>
</dependency>
<dependency>
<groupId>com.google.auto.value</groupId>
<artifactId>auto-value-annotations</artifactId>
<version>${auto-value.version}</version>
</dependency>
<dependency>
<groupId>org.graylog2</groupId>
<artifactId>graylog2-server</artifactId>
@ -127,24 +118,6 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-jdk14</artifactId>
<version>1.7.21</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
<version>${guice.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.google.inject.extensions</groupId>
<artifactId>guice-assistedinject</artifactId>
<version>${guice.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.google.auto.value</groupId>
<artifactId>auto-value</artifactId>
@ -173,20 +146,43 @@
</resource>
</resources>
<plugins>
<plugin>
<groupId>de.thetaphi</groupId>
<artifactId>forbiddenapis</artifactId>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<groupId>com.mycila</groupId>
<artifactId>license-maven-plugin</artifactId>
<configuration>
<skip>true</skip>
<licenseSets>
<licenseSet>
<header>com/mycila/maven/plugin/license/templates/SSPL-1.txt</header>
<properties>
<project.inceptionYear>2024</project.inceptionYear>
<owner>johan@nosd.in</owner>
</properties>
<includes>
<include>**/src/main/java/**</include>
<include>**/src/test/java/**</include>
<include>**/pom.xml</include>
<include>*.js</include>
<include>src/web/**/*.js</include>
<include>src/web/**/*.jsx</include>
<include>src/web/**/*.ts</include>
<include>src/web/**/*.tsx</include>
<include>src/web/**/*.css</include>
</includes>
<excludes>
<exclude>*.config.js</exclude>
</excludes>
</licenseSet>
</licenseSets>
</configuration>
<executions>
<execution>
<goals>
<goal>check</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
@ -197,25 +193,12 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.13.0</version>
<version>3.10.1</version>
<configuration>
<release>17</release>
<showWarnings>true</showWarnings>
<showDeprecation>true</showDeprecation>
<forceJavacCompilerUse>false</forceJavacCompilerUse>
<fork>true</fork>
<compilerArgs>
<arg>-J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED</arg>
<arg>-J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED</arg>
<arg>-J--add-opens=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED</arg>
<arg>-J--add-opens=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED</arg>
<arg>-J--add-opens=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED</arg>
<arg>-J--add-opens=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED</arg>
<arg>-J--add-opens=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED</arg>
<arg>-J--add-opens=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED</arg>
<arg>-J--add-opens=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED</arg>
<arg>-J--add-opens=jdk.compiler/com.sun.tools.javac.jvm=ALL-UNNAMED</arg>
</compilerArgs>
<forceJavacCompilerUse>true</forceJavacCompilerUse>
<annotationProcessors>
<annotationProcessor>com.google.auto.value.processor.AutoValueProcessor</annotationProcessor>
</annotationProcessors>
@ -241,12 +224,6 @@
<version>2.4.1</version>
<configuration>
<minimizeJar>false</minimizeJar>
<artifactSet>
<excludes>
<exclude>com.fasterxml.jackson.core:*</exclude>
<exclude>joda-time:joda-time</exclude>
</excludes>
</artifactSet>
</configuration>
<executions>
<execution>
@ -345,7 +322,6 @@
<plugin>
<groupId>com.github.eirslett</groupId>
<artifactId>frontend-maven-plugin</artifactId>
<version>1.15.0</version>
<executions>
<execution>

View File

@ -30,11 +30,11 @@ import java.util.Collections;
public class RedisLookupPlugin implements Plugin {
@Override
public PluginMetaData metadata() {
return new RedisLookupMetaData();
return new RedisLookupPluginMetaData();
}
@Override
public Collection<PluginModule> modules () {
return Collections.<PluginModule>singletonList(new RedisLookupModule());
return Collections.<PluginModule>singletonList(new RedisLookupPluginModule());
}
}

View File

@ -27,17 +27,17 @@ import java.util.Set;
/**
* Implement the PluginMetaData interface here.
*/
public class RedisLookupMetaData implements PluginMetaData {
public class RedisLookupPluginMetaData implements PluginMetaData {
private static final String PLUGIN_PROPERTIES = "in.nosd.redis.graylog-plugin-redis-lookup/graylog-plugin.properties";
@Override
public String getUniqueId() {
return "in.nosd.redis.RedisLookupPlugin";
return "in.nosd.redis.RedisLookupPluginPlugin";
}
@Override
public String getName() {
return "Redis Lookup";
return "RedisLookupPlugin";
}
@Override

View File

@ -16,17 +16,17 @@
*/
package in.nosd.redis;
import com.google.inject.Binder;
import com.google.inject.TypeLiteral;
import com.google.inject.multibindings.MapBinder;
import org.graylog.plugins.pipelineprocessor.ast.functions.Function;
import org.graylog2.plugin.PluginConfigBean;
import org.graylog2.plugin.PluginModule;
import com.google.inject.multibindings.MapBinder;
import com.google.inject.Binder;
import com.google.inject.TypeLiteral;
import in.nosd.redis.dataadapters.RedisLookupDataAdapter;
import in.nosd.redis.functions.RedisLookupPluginFunction;
import in.nosd.redis.migrations.V104_MigrateRedisType;
import java.util.Collections;
import java.util.Set;
@ -34,7 +34,7 @@ import java.util.Set;
/**
* Extend the PluginModule abstract class here to add you plugin to the system.
*/
public class RedisLookupModule extends PluginModule {
public class RedisLookupPluginModule extends PluginModule {
/**
* Returns all configuration beans required by this plugin.
*
@ -56,7 +56,7 @@ public class RedisLookupModule extends PluginModule {
* addMessageFilter(Class<? extends MessageFilter>);
* addMessageOutput(Class<? extends MessageOutput>);
* addPeriodical(Class<? extends Periodical>);
// * addAlarmCallback(Class<? extends AlarmCallback>);
* addAlarmCallback(Class<? extends AlarmCallback>);
* addInitializer(Class<? extends Service>);
* addRestResource(Class<? extends PluginRestResource>);
*
@ -66,11 +66,13 @@ public class RedisLookupModule extends PluginModule {
* addConfigBeans();
*/
addMigration(V104_MigrateRedisType.class);
addMessageProcessorFunction(RedisLookupPluginFunction.NAME, RedisLookupPluginFunction.class);
installLookupDataAdapter2(RedisLookupDataAdapter.NAME, RedisLookupDataAdapter.class,
RedisLookupDataAdapter.Factory.class, RedisLookupDataAdapter.Config.class);
addConfigBeans();
}

View File

@ -0,0 +1,39 @@
/*
* Copyright (C) 2024 johan@nosd.in
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
* as published by MongoDB, Inc.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Server Side Public License for more details.
*
* You should have received a copy of the Server Side Public License
* along with this program. If not, see
* <http://www.mongodb.com/licensing/server-side-public-license>.
*/
package in.nosd.redis;
import org.graylog2.plugin.Plugin;
import org.graylog2.plugin.PluginMetaData;
import org.graylog2.plugin.PluginModule;
import java.util.Collection;
import java.util.Collections;
/**
* Implement the Plugin interface here.
*/
public class RedisLookupPluginPlugin implements Plugin {
@Override
public PluginMetaData metadata() {
return new RedisLookupPluginMetaData();
}
@Override
public Collection<PluginModule> modules () {
return Collections.<PluginModule>singletonList(new RedisLookupPluginModule());
}
}

View File

@ -16,20 +16,6 @@
*/
package in.nosd.redis.dataadapters;
import java.util.List;
import java.util.Optional;
import org.apache.commons.lang3.StringUtils;
import org.graylog.autovalue.WithBeanGetter;
import org.graylog2.lookup.dto.DataAdapterDto;
import org.graylog2.plugin.lookup.LookupCachePurge;
import org.graylog2.plugin.lookup.LookupDataAdapter;
import org.graylog2.plugin.lookup.LookupDataAdapterConfiguration;
import org.graylog2.plugin.lookup.LookupResult;
import org.joda.time.Duration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer;
@ -40,23 +26,41 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.google.auto.value.AutoValue;
import org.graylog2.lookup.dto.DataAdapterDto;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Multimap;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
import org.apache.commons.lang3.StringUtils;
import io.lettuce.core.RedisClient;
import io.lettuce.core.RedisURI;
import io.lettuce.core.api.StatefulRedisConnection;
import io.lettuce.core.api.sync.RedisCommands;
import jakarta.validation.constraints.Min;
import jakarta.validation.constraints.NotEmpty;
import reactor.util.annotation.Nullable;
import in.nosd.redis.dataadapters.AutoValue_RedisLookupDataAdapter_Config;
import org.graylog2.plugin.lookup.LookupCachePurge;
import org.graylog2.plugin.lookup.LookupDataAdapter;
import org.graylog2.plugin.lookup.LookupDataAdapterConfiguration;
import org.graylog2.plugin.lookup.LookupResult;
import org.joda.time.Duration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.Nullable;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
import javax.validation.constraints.Min;
import javax.validation.constraints.NotEmpty;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.StringJoiner;
public class RedisLookupDataAdapter extends LookupDataAdapter {
private static final Logger LOG = LoggerFactory.getLogger(RedisLookupDataAdapter.class);
// Should be same as lookupTableAdapters type in index.jsx
public static final String NAME = "RedisLookup";
private static final Duration REFRESH_INTERVAL_DURATION = Duration.ZERO;
@ -86,8 +90,8 @@ public class RedisLookupDataAdapter extends LookupDataAdapter {
public RedisLookupDataAdapter(@Assisted("dto") DataAdapterDto dto,
MetricRegistry metricRegistry) {
super(dto, metricRegistry);
this.config = (Config) dto.config();
this.config = (Config) dto.config();
RedisURI redisUri ;
redisUri = RedisURI.Builder.redis(this.config.redisHost(),this.config.redisPort())
.withPort(this.config.redisPort())
@ -183,6 +187,7 @@ public class RedisLookupDataAdapter extends LookupDataAdapter {
// This is deprecated, see setValue
@Deprecated
public void set(Object key, Object value) {
return;
}
@Override
@ -190,7 +195,6 @@ public class RedisLookupDataAdapter extends LookupDataAdapter {
return setValueWithTtl(key, value, this.config.redisKeyTTL());
}
@Override
public LookupResult setValueWithTtl(Object key, Object value, Long ttlSec) {
final Timer.Context time = redisSetRequestTimer.time();
final String trimmedKey = StringUtils.trimToNull(key.toString());
@ -269,7 +273,6 @@ public class RedisLookupDataAdapter extends LookupDataAdapter {
}
// TTL -1 = never expire
@Override
public LookupResult assignTtl(Object key, Long ttlSec) {
final Timer.Context time = redisAssignTtlRequestTimer.time();
final String trimmedKey = StringUtils.trimToNull(key.toString());
@ -344,12 +347,11 @@ public class RedisLookupDataAdapter extends LookupDataAdapter {
}
}
@Override
public LookupResult setStringListWithTtl(Object key, List<String> listValue, Long ttlSec) {
final Timer.Context time = redisSetStringListWithTtlRequestTimer.time();
try {
setStringList(key, listValue);
return assignTtl(key, ttlSec);
return(assignTtl(key, ttlSec));
} catch (Exception e) {
// This exception comes from assignTtl
if (e.getMessage().startsWith("WRONGTYPE Operation against a key holding the wrong kind of value")) {
@ -408,6 +410,7 @@ public class RedisLookupDataAdapter extends LookupDataAdapter {
.redisHost("127.0.0.1")
.redisPort(6379)
.redisDB(0)
.redisType("strings")
.redisKeyTTL(-1)
.redisUsername("")
.redisPassword("")
@ -418,12 +421,11 @@ public class RedisLookupDataAdapter extends LookupDataAdapter {
@AutoValue
@WithBeanGetter
@JsonAutoDetect
@JsonDeserialize(builder = AutoValue_RedisLookupDataAdapter_Config.Builder.class)
@JsonDeserialize(builder = RedisLookupDataAdapter.Config.Builder.class)
@JsonTypeName(NAME)
@JsonInclude(JsonInclude.Include.NON_EMPTY)
public static abstract class Config implements LookupDataAdapterConfiguration {
public abstract static class Config implements LookupDataAdapterConfiguration {
@Override
@JsonProperty(TYPE_FIELD)
@ -440,6 +442,13 @@ public class RedisLookupDataAdapter extends LookupDataAdapter {
@JsonProperty("redis_database")
@Min(0)
public abstract int redisDB();
@JsonProperty("redis_type")
/* FIXME: This should be notEmpty, but migration crash when dbDataAdapterService.findAll() with error
* "Missing required properties: redisType"
* so dont flag NoteEmpty and put Nullable. */
@Nullable
public abstract String redisType();
@JsonProperty("redis_ttl")
@Min(0)
@ -471,6 +480,11 @@ public class RedisLookupDataAdapter extends LookupDataAdapter {
@AutoValue.Builder
public abstract static class Builder {
@JsonCreator
public static Builder create() {
return Config.builder();
}
@JsonProperty(TYPE_FIELD)
public abstract Builder type(String type);
@ -482,6 +496,9 @@ public class RedisLookupDataAdapter extends LookupDataAdapter {
@JsonProperty("redis_database")
public abstract Builder redisDB(int redisDB);
@JsonProperty("redis_type")
public abstract Builder redisType(String redisType);
@JsonProperty("redis_ttl")
public abstract Builder redisKeyTTL(long redisKeyTTL);

View File

@ -30,7 +30,7 @@ import org.graylog2.plugin.lookup.LookupResult;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.inject.Inject;
import javax.inject.Inject;
public class RedisLookupPluginFunction extends LookupTableFunction<GenericLookupResult> {

View File

@ -0,0 +1,140 @@
/*
* Copyright (C) 2024 johan@nosd.in
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
* as published by MongoDB, Inc.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Server Side Public License for more details.
*
* You should have received a copy of the Server Side Public License
* along with this program. If not, see
* <http://www.mongodb.com/licensing/server-side-public-license>.
*/
package in.nosd.redis.migrations;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.auto.value.AutoValue;
import com.google.common.collect.ImmutableSet;
import org.graylog.autovalue.WithBeanGetter;
import in.nosd.redis.dataadapters.RedisLookupDataAdapter;
import in.nosd.redis.dataadapters.RedisLookupDataAdapter.Config;
import org.graylog2.events.ClusterEventBus;
import org.graylog2.lookup.db.DBDataAdapterService;
import org.graylog2.lookup.dto.DataAdapterDto;
import org.graylog2.lookup.events.DataAdaptersUpdated;
import org.graylog2.migrations.Migration;
import org.graylog2.plugin.cluster.ClusterConfigService;
import org.graylog2.plugin.lookup.LookupDataAdapterConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import java.time.ZonedDateTime;
import java.util.Collections;
import java.util.Collection;
import java.util.Set;
import java.util.stream.Collectors;
import static com.google.common.base.Strings.isNullOrEmpty;
// From graylog-plugin-threatintel/src/main/java/org/graylog/plugins/threatintel/migrations/V20170821100300_MigrateOTXAPIToken.java
/* Manually :
*
* Go into mongodb instance
* Locate "lut_data_adapters" table
* Edit every entry for which "config.type" == "RedisLookup"
* Add "redis_type" key with value "strings"
*/
public class V104_MigrateRedisType extends Migration {
private static final Logger LOG = LoggerFactory.getLogger(V104_MigrateRedisType.class);
private static final ImmutableSet<String> REDIS_DATA_ADAPTER_NAMES = ImmutableSet.of("redis-lookup");
private final ClusterConfigService clusterConfigService;
private final DBDataAdapterService dbDataAdapterService;
private final ClusterEventBus clusterBus;
@Inject
public V104_MigrateRedisType(ClusterConfigService clusterConfigService,
DBDataAdapterService dbDataAdapterService,
ClusterEventBus clusterBus) {
this.clusterConfigService = clusterConfigService;
this.dbDataAdapterService = dbDataAdapterService;
this.clusterBus = clusterBus;
}
@Override
public ZonedDateTime createdAt() {
return ZonedDateTime.parse("2024-03-03T15:42:42Z");
}
@Override
public void upgrade() {
Config redisConf;
// List every instance of lookup DB adapter, and find instance with Config.type == "RedisLookup" (RedisLookupDataAdapter.NAME)
Collection<DataAdapterDto> DataAdapters = dbDataAdapterService.findAll();
for (DataAdapterDto dataAdapterDto : DataAdapters) {
if (dataAdapterDto.config().type().equals(RedisLookupDataAdapter.NAME)) {
redisConf = (Config) dataAdapterDto.config();
final Config newConf = Config.builder()
.type(redisConf.type())
.redisHost(redisConf.redisHost())
.redisPort(redisConf.redisPort())
.redisDB(redisConf.redisDB())
.redisType("strings")
.redisKeyTTL(redisConf.redisKeyTTL())
.redisUsername(redisConf.redisUsername())
.redisPassword(redisConf.redisPassword())
.build();
final DataAdapterDto newDto = DataAdapterDto.builder()
.id(dataAdapterDto.id())
.config(newConf)
.title(dataAdapterDto.title())
.description(dataAdapterDto.description())
.name(dataAdapterDto.name())
.build();
final DataAdapterDto saved = dbDataAdapterService.save(newDto);
clusterBus.post(DataAdaptersUpdated.create(saved.id()));
LOG.debug("Redis data adapter <{}> migrated: data_type added", dataAdapterDto.name());
}
}
}
@JsonAutoDetect
@AutoValue
@WithBeanGetter
public static abstract class MigrationCompleted {
@JsonProperty("converted_redis_type")
public abstract boolean convertedRedisType();
@JsonProperty("data_adapter_ids")
public abstract Set<String> dataAdapterIds();
@JsonCreator
public static MigrationCompleted create(@JsonProperty("data_adapter_ids") final Set<String> dataAdapterIds,
@JsonProperty("converted_redis_type") final boolean convertedRedisType) {
return new AutoValue_V104_MigrateRedisType_MigrationCompleted(convertedRedisType, dataAdapterIds);
}
public static MigrationCompleted convertedType(@JsonProperty("data_adapter_ids") final Set<String> dataAdapterIds) {
return create(dataAdapterIds, true);
}
public static MigrationCompleted notConvertedType() {
return create(Collections.emptySet(), false);
}
}
}

View File

@ -1 +1 @@
in.nosd.redis.RedisLookupPlugin
in.nosd.redis.RedisLookupPluginPlugin

View File

@ -1,35 +1,36 @@
/*
* Copyright (C) 2024 johan@nosd.in
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
* as published by MongoDB, Inc.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Server Side Public License for more details.
*
* You should have received a copy of the Server Side Public License
* along with this program. If not, see
* <http://www.mongodb.com/licensing/server-side-public-license>.
*/
import React from 'react';
class RedisLookupAdapterDocumentation extends React.Component {
render() {
const style = { marginBottom: 10 };
return (
<div>
<p style={style}>
The Redis Lookup data adapter lookup redis for the given key and returns the values.<br/>
It supports writing key/values to Redis (SET command). <br/>
All created keys will have the TTL configured for the data adapter.
</p>
</div>
)
;
}
}
export default RedisLookupAdapterDocumentation;
/*
* Copyright (C) 2024 johan@nosd.in
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
* as published by MongoDB, Inc.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Server Side Public License for more details.
*
* You should have received a copy of the Server Side Public License
* along with this program. If not, see
* <http://www.mongodb.com/licensing/server-side-public-license>.
*/
/* eslint-disable react/no-unescaped-entities, no-template-curly-in-string */
import React from 'react';
class RedisLookupAdapterDocumentation extends React.Component {
render() {
const style = { marginBottom: 10 };
return (
<div>
<p style={style}>
The Redis Lookup data adapter lookup redis for the given key and returns the values.<br/>
It supports writing key/values to Redis (SET command). <br/>
All created keys will have the TTL configured for the data adapter.
</p>
</div>
)
;
}
}
export default RedisLookupAdapterDocumentation;

View File

@ -1,115 +1,138 @@
/*
* Copyright (C) 2024 johan@nosd.in
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
* as published by MongoDB, Inc.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Server Side Public License for more details.
*
* You should have received a copy of the Server Side Public License
* along with this program. If not, see
* <http://www.mongodb.com/licensing/server-side-public-license>.
*/
import React, { useCallback, useEffect, useRef, useState } from 'react';
import PropTypes from 'prop-types';
//import { Button } from 'components/graylog';
import { Input } from 'components/bootstrap';
class RedisLookupAdapterFieldSet extends React.Component {
static propTypes = {
config: PropTypes.shape({
redis_host: PropTypes.string.isRequired,
}).isRequired,
updateConfig: PropTypes.func.isRequired,
handleFormEvent: PropTypes.func.isRequired,
validationState: PropTypes.func.isRequired,
validationMessage: PropTypes.func.isRequired,
};
handleSelect = (fieldName) => {
return (selectedIndicator) => {
const config = lodash.cloneDeep(this.props.config);
config[fieldName] = selectedIndicator;
this.props.updateConfig(config);
};
};
render() {
const { config } = this.props;
return (
<fieldset>
<Input type="text"
id="redis_host"
name="redis_host"
label="Redis host"
required
onChange={this.props.handleFormEvent}
help={this.props.validationMessage('redis_host', 'Your Redis Host')}
bsStyle={this.props.validationState('redis_host')}
value={config.redis_host}
labelClassName="col-sm-3"
wrapperClassName="col-sm-9" />
<Input type="text"
id="redis_port"
name="redis_port"
label="Redis port"
required
onChange={this.props.handleFormEvent}
help={this.props.validationMessage('redis_port', 'Redis port instance is listening on')}
bsStyle={this.props.validationState('redis_port')}
value={config.redis_port}
labelClassName="col-sm-3"
wrapperClassName="col-sm-9" />
<Input type="text"
id="redis_database"
name="redis_database"
label="Redis database"
required
onChange={this.props.handleFormEvent}
help={this.props.validationMessage('redis_database', 'Redis database')}
bsStyle={this.props.validationState('redis_database')}
value={config.redis_database}
labelClassName="col-sm-3"
wrapperClassName="col-sm-9" />
<Input type="text"
id="redis_ttl"
name="redis_ttl"
label="Redis key TTL"
required
onChange={this.props.handleFormEvent}
help={this.props.validationMessage('redis_ttl', 'Redis key TTL in seconds. Set -1 to not expire keys')}
bsStyle={this.props.validationState('redis_ttl')}
value={config.redis_ttl}
labelClassName="col-sm-3"
wrapperClassName="col-sm-9" />
<Input type="text"
id="redis_username"
name="redis_username"
label="Redis username"
onChange={this.props.handleFormEvent}
help={this.props.validationMessage('redis_username', 'Redis username. Leave empty for no auth or "requirepass" authentication')}
bsStyle={this.props.validationState('redis_username')}
value={config.redis_username}
labelClassName="col-sm-3"
wrapperClassName="col-sm-9" />
<Input type="password"
id="redis_password"
label="Redis password"
onChange={this.props.handleFormEvent}
help={this.props.validationMessage('redis_password', 'Redis password. Leave empty for no auth')}
bsStyle={this.props.validationState('redis_password')}
value={config.redis_password}
labelClassName="col-sm-3"
wrapperClassName="col-sm-9">
</Input>
</fieldset>
);
}
}
export default RedisLookupAdapterFieldSet;
/*
* Copyright (C) 2024 johan@nosd.in
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
* as published by MongoDB, Inc.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Server Side Public License for more details.
*
* You should have received a copy of the Server Side Public License
* along with this program. If not, see
* <http://www.mongodb.com/licensing/server-side-public-license>.
*/
import React, { useCallback, useEffect, useRef, useState } from 'react';
import PropTypes from 'prop-types';
//import ObjectUtils from 'util/ObjectUtils';
import { Input } from 'components/bootstrap';
// "Missing or invalid plugin" in "dataAdapter create/Data adapter type" list when imported
//import { Select } from './components/common';
//import { Select } from '../../../../graylog2-server/graylog2-web-interface/src/components/common';
class RedisLookupAdapterFieldSet extends React.Component {
static propTypes = {
config: PropTypes.shape({
redis_host: PropTypes.string.isRequired,
redis_port: PropTypes.number.isRequired,
redis_database: PropTypes.number.isRequired,
redis_type: PropTypes.string.isRequired,
redis_ttl: PropTypes.number.isRequired,
redis_username: PropTypes.string,
redis_password: PropTypes.string,
}).isRequired,
updateConfig: PropTypes.func.isRequired,
handleFormEvent: PropTypes.func.isRequired,
validationState: PropTypes.func.isRequired,
validationMessage: PropTypes.func.isRequired,
};
/* _onRedisTypeSelect = (type) => {
const { config, updateConfig } = this.props;
const newConfig = ObjectUtils.clone(config);
newConfig.redis_type = type;
updateConfig(newConfig);
};
*/
render() {
const { config } = this.props;
return (
<fieldset>
<Input type="text"
id="redis_host"
name="redis_host"
label="Redis host"
required
onChange={this.props.handleFormEvent}
help={this.props.validationMessage('redis_host', 'Your Redis Host')}
bsStyle={this.props.validationState('redis_host')}
value={config.redis_host}
labelClassName="col-sm-3"
wrapperClassName="col-sm-9" />
<Input type="text"
id="redis_port"
name="redis_port"
label="Redis port"
required
onChange={this.props.handleFormEvent}
help={this.props.validationMessage('redis_port', 'Redis port instance is listening on')}
bsStyle={this.props.validationState('redis_port')}
value={config.redis_port}
labelClassName="col-sm-3"
wrapperClassName="col-sm-9" />
<Input type="text"
id="redis_database"
name="redis_database"
label="Redis database"
required
onChange={this.props.handleFormEvent}
help={this.props.validationMessage('redis_database', 'Redis database')}
bsStyle={this.props.validationState('redis_database')}
value={config.redis_database}
labelClassName="col-sm-3"
wrapperClassName="col-sm-9" />
<Input type="text"
id="redis_type"
name="redis_type"
label="Redis data type"
required
onChange={this.props.handleFormEvent}
help={this.props.validationMessage('redis_type', 'Redis data type used for lookups. Should be one of "strings", "streams".')}
bsStyle={this.props.validationState('redis_type')}
value={config.redis_type}
labelClassName="col-sm-3"
wrapperClassName="col-sm-9">
</Input>
<Input type="text"
id="redis_ttl"
name="redis_ttl"
label="Redis key TTL"
required
onChange={this.props.handleFormEvent}
help={this.props.validationMessage('redis_ttl', 'Redis key TTL in seconds. Set -1 to not expire keys')}
bsStyle={this.props.validationState('redis_ttl')}
value={config.redis_ttl}
labelClassName="col-sm-3"
wrapperClassName="col-sm-9" />
<Input type="text"
id="redis_username"
name="redis_username"
label="Redis username"
onChange={this.props.handleFormEvent}
help={this.props.validationMessage('redis_username', 'Redis username. Leave empty for no auth or "requirepass" authentication')}
bsStyle={this.props.validationState('redis_username')}
value={config.redis_username}
labelClassName="col-sm-3"
wrapperClassName="col-sm-9" />
<Input type="password"
id="redis_password"
label="Redis password"
onChange={this.props.handleFormEvent}
help={this.props.validationMessage('redis_password', 'Redis password. Leave empty for no auth')}
bsStyle={this.props.validationState('redis_password')}
value={config.redis_password}
labelClassName="col-sm-3"
wrapperClassName="col-sm-9">
</Input>
</fieldset>
);
}
}
export default RedisLookupAdapterFieldSet;

View File

@ -1,52 +1,54 @@
/*
* Copyright (C) 2024 johan@nosd.in
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
* as published by MongoDB, Inc.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Server Side Public License for more details.
*
* You should have received a copy of the Server Side Public License
* along with this program. If not, see
* <http://www.mongodb.com/licensing/server-side-public-license>.
*/
// eslint-disable-next-line react/prefer-stateless-function
import React from 'react';
import PropTypes from 'prop-types';
class RedisLookupAdapterSummary extends React.Component {
static propTypes = {
dataAdapter: PropTypes.shape({
config: PropTypes.shape({
redis_host: PropTypes.string.isRequired,
}),
}),
};
render() {
const { config } = this.props.dataAdapter;
return (
<dl>
<dt>Redis host</dt>
<dd>{config.redis_host || 'n/a'}</dd>
<dt>Redis port</dt>
<dd>{config.redis_port || 'n/a'}</dd>
<dt>Redis database</dt>
<dd>{config.redis_database}</dd>
<dt>Redis key TTL</dt>
<dd>{config.redis_ttl || 'n/a'}</dd>
<dt>Redis username</dt>
<dd>{config.redis_username || 'n/a'}</dd>
<dt>Redis password</dt>
<dd>******</dd>
</dl>
);
}
}
export default RedisLookupAdapterSummary;
/*
* Copyright (C) 2024 johan@nosd.in
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
* as published by MongoDB, Inc.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Server Side Public License for more details.
*
* You should have received a copy of the Server Side Public License
* along with this program. If not, see
* <http://www.mongodb.com/licensing/server-side-public-license>.
*/
// eslint-disable-next-line react/prefer-stateless-function
import React from 'react';
import PropTypes from 'prop-types';
class RedisLookupAdapterSummary extends React.Component {
static propTypes = {
dataAdapter: PropTypes.shape({
config: PropTypes.shape({
redis_host: PropTypes.string.isRequired,
}),
}),
};
render() {
const { config } = this.props.dataAdapter;
return (
<dl>
<dt>Redis host</dt>
<dd>{config.redis_host || 'n/a'}</dd>
<dt>Redis port</dt>
<dd>{config.redis_port || 'n/a'}</dd>
<dt>Redis database</dt>
<dd>{config.redis_database}</dd>
<dt>Redis key TTL</dt>
<dd>{config.redis_ttl || 'n/a'}</dd>
<dt>Redis data type</dt>
<dd>{config.redis_type || 'n/a'}</dd>
<dt>Redis username</dt>
<dd>{config.redis_username || 'n/a'}</dd>
<dt>Redis password</dt>
<dd>******</dd>
</dl>
);
}
}
export default RedisLookupAdapterSummary;

View File

@ -1,39 +1,39 @@
/*
* Copyright (C) 2024 johan@nosd.in
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
* as published by MongoDB, Inc.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Server Side Public License for more details.
*
* You should have received a copy of the Server Side Public License
* along with this program. If not, see
* <http://www.mongodb.com/licensing/server-side-public-license>.
*/
// eslint-disable-next-line no-unused-vars
import webpackEntry from 'webpack-entry';
import { PluginManifest, PluginStore } from 'graylog-web-plugin/plugin';
import packageJson from '../../package.json';
import RedisLookupAdapterDocumentation from './dataadapters/RedisLookupAdapterDocumentation';
import RedisLookupAdapterFieldSet from './dataadapters/RedisLookupAdapterFieldSet';
import RedisLookupAdapterSummary from './dataadapters/RedisLookupAdapterSummary';
const manifest = new PluginManifest(packageJson, {
lookupTableAdapters: [
{
type: 'RedisLookup',
displayName: 'Redis Lookup',
formComponent: RedisLookupAdapterFieldSet,
summaryComponent: RedisLookupAdapterSummary,
documentationComponent: RedisLookupAdapterDocumentation,
},
],
});
PluginStore.register(manifest);
/*
* Copyright (C) 2024 johan@nosd.in
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
* as published by MongoDB, Inc.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Server Side Public License for more details.
*
* You should have received a copy of the Server Side Public License
* along with this program. If not, see
* <http://www.mongodb.com/licensing/server-side-public-license>.
*/
// eslint-disable-next-line no-unused-vars
import webpackEntry from 'webpack-entry';
import { PluginManifest, PluginStore } from 'graylog-web-plugin/plugin';
import packageJson from '../../package.json';
import RedisLookupAdapterDocumentation from './dataadapters/RedisLookupAdapterDocumentation';
import RedisLookupAdapterFieldSet from './dataadapters/RedisLookupAdapterFieldSet';
import RedisLookupAdapterSummary from './dataadapters/RedisLookupAdapterSummary';
const manifest = new PluginManifest(packageJson, {
lookupTableAdapters: [
{
type: 'RedisLookup',
displayName: 'Redis Lookup',
formComponent: RedisLookupAdapterFieldSet,
summaryComponent: RedisLookupAdapterSummary,
documentationComponent: RedisLookupAdapterDocumentation,
},
],
});
PluginStore.register(manifest);

View File

@ -19,6 +19,6 @@ const { PluginWebpackConfig } = require('graylog-web-plugin');
const { loadBuildConfig } = require('graylog-web-plugin');
// Remember to use the same name here and in `getUniqueId()` in the java MetaData class
module.exports = new PluginWebpackConfig(__dirname, 'in.nosd.redis.RedisLookupPlugin', loadBuildConfig(path.resolve(__dirname, './build.config')), {
module.exports = new PluginWebpackConfig(__dirname, 'in.nosd.redis.RedisLookupPluginPlugin', loadBuildConfig(path.resolve(__dirname, './build.config')), {
// Here goes your additional webpack configuration.
});

5481
yarn.lock

File diff suppressed because it is too large Load Diff