first commit

This commit is contained in:
2024-02-03 14:07:41 +01:00
commit 1ff3f557a7
39 changed files with 165488 additions and 0 deletions

View File

@ -0,0 +1,40 @@
/*
* Copyright (C) 2020 Graylog, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
* as published by MongoDB, Inc.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Server Side Public License for more details.
*
* You should have received a copy of the Server Side Public License
* along with this program. If not, see
* <http://www.mongodb.com/licensing/server-side-public-license>.
*/
package in.nosd.redis;
import org.graylog2.plugin.Plugin;
import org.graylog2.plugin.PluginMetaData;
import org.graylog2.plugin.PluginModule;
import java.util.Collection;
import java.util.Collections;
/**
* This is the plugin. Your class should implement one of the existing plugin
* interfaces. (i.e. AlarmCallback, MessageInput, MessageOutput)
*/
public class RedisLookupPlugin implements Plugin {
@Override
public PluginMetaData metadata() {
return new RedisLookupPluginMetaData();
}
@Override
public Collection<PluginModule> modules () {
return Collections.<PluginModule>singletonList(new RedisLookupPluginModule());
}
}

View File

@ -0,0 +1,73 @@
/*
* Copyright (C) 2020 Graylog, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
* as published by MongoDB, Inc.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Server Side Public License for more details.
*
* You should have received a copy of the Server Side Public License
* along with this program. If not, see
* <http://www.mongodb.com/licensing/server-side-public-license>.
*/
package in.nosd.redis;
import org.graylog2.plugin.PluginMetaData;
import org.graylog2.plugin.ServerStatus;
import org.graylog2.plugin.Version;
import java.net.URI;
import java.util.Collections;
import java.util.Set;
/**
* Implement the PluginMetaData interface here.
*/
public class RedisLookupPluginMetaData implements PluginMetaData {
private static final String PLUGIN_PROPERTIES = "in.nosd.redis.graylog-plugin-redis-lookup/graylog-plugin.properties";
@Override
public String getUniqueId() {
return "in.nosd.redis.RedisLookupPluginPlugin";
}
@Override
public String getName() {
return "RedisLookupPlugin";
}
@Override
public String getAuthor() {
return "johan <johan@nosd.in>";
}
@Override
public URI getURL() {
return URI.create("https://github.com/https://git.nosd.in/yo/graylog-plugin-redis-lookup");
}
@Override
public Version getVersion() {
return Version.fromPluginProperties(getClass(), PLUGIN_PROPERTIES, "version", Version.from(0, 0, 0, "unknown"));
}
@Override
public String getDescription() {
// TODO Insert correct plugin description
return "Redis database lookup & write functions for the Graylog Pipeline Processor";
}
@Override
public Version getRequiredVersion() {
return Version.fromPluginProperties(getClass(), PLUGIN_PROPERTIES, "graylog.version", Version.from(0, 0, 0, "unknown"));
}
@Override
public Set<ServerStatus.Capability> getRequiredCapabilities() {
return Collections.emptySet();
}
}

View File

@ -0,0 +1,87 @@
/*
* Copyright (C) 2020 Graylog, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
* as published by MongoDB, Inc.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Server Side Public License for more details.
*
* You should have received a copy of the Server Side Public License
* along with this program. If not, see
* <http://www.mongodb.com/licensing/server-side-public-license>.
*/
package in.nosd.redis;
import com.google.inject.Binder;
import com.google.inject.TypeLiteral;
import com.google.inject.multibindings.MapBinder;
import org.graylog.plugins.pipelineprocessor.ast.functions.Function;
import org.graylog2.plugin.PluginConfigBean;
import org.graylog2.plugin.PluginModule;
import in.nosd.redis.dataadapters.RedisLookupDataAdapter;
import in.nosd.redis.functions.RedisLookupPluginFunction;
import java.util.Collections;
import java.util.Set;
/**
* Extend the PluginModule abstract class here to add you plugin to the system.
*/
public class RedisLookupPluginModule extends PluginModule {
/**
* Returns all configuration beans required by this plugin.
*
* Implementing this method is optional. The default method returns an empty {@link Set}.
*/
@Override
public Set<? extends PluginConfigBean> getConfigBeans() {
return Collections.emptySet();
}
@Override
protected void configure() {
/*
* Register your plugin types here.
*
* Examples:
*
* addMessageInput(Class<? extends MessageInput>);
* addMessageFilter(Class<? extends MessageFilter>);
* addMessageOutput(Class<? extends MessageOutput>);
* addPeriodical(Class<? extends Periodical>);
* addAlarmCallback(Class<? extends AlarmCallback>);
* addInitializer(Class<? extends Service>);
* addRestResource(Class<? extends PluginRestResource>);
*
*
* Add all configuration beans returned by getConfigBeans():
*
* addConfigBeans();
*/
addMessageProcessorFunction(RedisLookupPluginFunction.NAME, RedisLookupPluginFunction.class);
installLookupDataAdapter2(RedisLookupDataAdapter.NAME, RedisLookupDataAdapter.class,
RedisLookupDataAdapter.Factory.class, RedisLookupDataAdapter.Config.class);
addConfigBeans();
}
private void addMessageProcessorFunction(String name, Class<? extends Function<?>> functionClass) {
addMessageProcessorFunction(binder(), name, functionClass);
}
private MapBinder<String, Function<?>> processorFunctionBinder(Binder binder) {
return MapBinder.newMapBinder(binder, TypeLiteral.get(String.class), new TypeLiteral<Function<?>>() {});
}
private void addMessageProcessorFunction(Binder binder, String name, Class<? extends Function<?>> functionClass) {
processorFunctionBinder(binder).addBinding(name).to(functionClass);
}
}

View File

@ -0,0 +1,39 @@
/*
* Copyright (C) 2020 Graylog, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
* as published by MongoDB, Inc.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Server Side Public License for more details.
*
* You should have received a copy of the Server Side Public License
* along with this program. If not, see
* <http://www.mongodb.com/licensing/server-side-public-license>.
*/
package in.nosd.redis;
import org.graylog2.plugin.Plugin;
import org.graylog2.plugin.PluginMetaData;
import org.graylog2.plugin.PluginModule;
import java.util.Collection;
import java.util.Collections;
/**
* Implement the Plugin interface here.
*/
public class RedisLookupPluginPlugin implements Plugin {
@Override
public PluginMetaData metadata() {
return new RedisLookupPluginMetaData();
}
@Override
public Collection<PluginModule> modules () {
return Collections.<PluginModule>singletonList(new RedisLookupPluginModule());
}
}

View File

@ -0,0 +1,288 @@
/*
* Copyright (C) 2020 Graylog, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
* as published by MongoDB, Inc.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Server Side Public License for more details.
*
* You should have received a copy of the Server Side Public License
* along with this program. If not, see
* <http://www.mongodb.com/licensing/server-side-public-license>.
*/
package in.nosd.redis.dataadapters;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.google.auto.value.AutoValue;
import org.graylog2.lookup.dto.DataAdapterDto;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Multimap;
import com.google.inject.assistedinject.Assisted;
import org.apache.commons.lang3.StringUtils;
import io.lettuce.core.RedisClient;
import io.lettuce.core.RedisURI;
import io.lettuce.core.api.StatefulRedisConnection;
import io.lettuce.core.api.sync.RedisCommands;
/*
//To delete after clean
import okhttp3.HttpUrl;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
import okhttp3.ResponseBody;
//END To delete after clean
*/
import in.nosd.redis.dataadapters.AutoValue_RedisLookupDataAdapter_Config;
import org.graylog2.plugin.lookup.LookupCachePurge;
import org.graylog2.plugin.lookup.LookupDataAdapter;
import org.graylog2.plugin.lookup.LookupDataAdapterConfiguration;
import org.graylog2.plugin.lookup.LookupResult;
import org.joda.time.Duration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.Nullable;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
import javax.validation.constraints.Min;
import javax.validation.constraints.NotEmpty;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.StringJoiner;
public class RedisLookupDataAdapter extends LookupDataAdapter {
private static final Logger LOG = LoggerFactory.getLogger(RedisLookupDataAdapter.class);
public static final String NAME = "RedisLookup";
private static final Duration REFRESH_INTERVAL_DURATION = Duration.ZERO;
private final Config config;
private final RedisClient client;
private RedisCommands<String, String> commands;
private final Timer redisGetRequestTimer;
private final Meter redisGetRequestErrors;
private final Timer redisSetRequestTimer;
private final Meter redisSetRequestErrors;
@Inject
public RedisLookupDataAdapter(@Assisted("dto") DataAdapterDto dto,
MetricRegistry metricRegistry) {
super(dto, metricRegistry);
this.config = (Config) dto.config();
RedisURI redisUri = RedisURI.Builder.redis(this.config.redisHost(),this.config.redisPort())
.withPort(this.config.redisPort())
//.withAuthentication(this.config.redisUsername(), this.config.redisPassword())
.withDatabase(this.config.redisDB())
.build();
this.client = RedisClient.create(redisUri);
/*this.client = RedisClient.create(RedisURI.Builder.redis(this.config.redisHost(),this.config.redisPort())
.withPort(this.config.redisPort())
.withAuthentication(this.config.redisUsername(), this.config.redisPassword())
.withDatabase(this.config.redisDB())
.build());
*/
this.redisGetRequestTimer = metricRegistry.timer(MetricRegistry.name(getClass(), "redisGetRequestTime"));
this.redisGetRequestErrors = metricRegistry.meter(MetricRegistry.name(getClass(), "redisGetRequestErrors"));
this.redisSetRequestTimer = metricRegistry.timer(MetricRegistry.name(getClass(), "redisSetRequestTime"));
this.redisSetRequestErrors = metricRegistry.meter(MetricRegistry.name(getClass(), "redisSetRequestErrors"));
}
// Add code to initialise Redis connection
@Override
protected void doStart() throws Exception {
StatefulRedisConnection<String, String> connection = this.client.connect();
this.commands = connection.sync();
}
// Add code to close Redis connection
@Override
protected void doStop() throws Exception {
}
@Override
public Duration refreshInterval() {
return REFRESH_INTERVAL_DURATION;
}
@Override
protected void doRefresh(LookupCachePurge cachePurge) throws Exception {
doStart();
cachePurge.purgeAll();
}
@Override
protected LookupResult doGet(Object key) {
final Timer.Context time = redisGetRequestTimer.time();
final String trimmedKey = StringUtils.trimToNull(key.toString());
if (trimmedKey == null) {
LOG.debug("A blank key was supplied");
return getEmptyResult();
}
try {
final String value = this.commands.get(trimmedKey);
if (value == null) {
LOG.warn("Redis GET request for key <{}> returned null, key do not exists.", trimmedKey);
redisGetRequestErrors.mark();
return LookupResult.empty();
}
return LookupResult.single(value);
//return LookupResult.single("coincoin");
} catch (Exception e) {
LOG.error("Redis GET request error for key <{}>", trimmedKey, e);
redisGetRequestErrors.mark();
return LookupResult.empty();
} finally {
time.stop();
}
}
@Override
public void set(Object key, Object value) {
//throw new UnsupportedOperationException();
LOG.warn("Entering Redis set function with {}={}", key.toString(), value.toString());
final Timer.Context time = redisSetRequestTimer.time();
try {
final String result = this.commands.set(key.toString(), value.toString());
if (!result.equals("OK")) {
LOG.warn("Redis SET key <{}> to value <{}> returned {}", key, value, result);
redisSetRequestErrors.mark();
return;
}
return;
} catch (Exception e) {
LOG.error("Redis SET key <{}> to value <{}> returned an exception: {}", key, value, e);
redisSetRequestErrors.mark();
return;
} finally {
time.stop();
}
}
public interface Factory extends LookupDataAdapter.Factory2<RedisLookupDataAdapter> {
@Override
RedisLookupDataAdapter create(@Assisted("dto") DataAdapterDto dto);
@Override
Descriptor getDescriptor();
}
public static class Descriptor extends LookupDataAdapter.Descriptor<Config> {
public Descriptor() {
super(NAME, Config.class);
}
@Override
public Config defaultConfiguration() {
return Config.builder()
.type(NAME)
.redisHost("127.0.0.1")
.redisPort(6379)
.redisDB(0)
/*.redisUsername("")
.redisPassword("")*/
.build();
}
}
@AutoValue
@JsonAutoDetect
@JsonDeserialize(builder = RedisLookupDataAdapter.Config.Builder.class)
@JsonTypeName(NAME)
@JsonInclude(JsonInclude.Include.NON_EMPTY)
public abstract static class Config implements LookupDataAdapterConfiguration {
@Override
@JsonProperty(TYPE_FIELD)
public abstract String type();
@JsonProperty("redis_host")
@NotEmpty
public abstract String redisHost();
@JsonProperty("redis_port")
@Min(1)
public abstract int redisPort();
@JsonProperty("redis_database")
@Min(0)
public abstract int redisDB();
/*@JsonProperty("redis_username")
@Nullable
public abstract String redisUsername();
@JsonProperty("redis_password")
@Nullable
public abstract String redisPassword();*/
public static Builder builder() {
return new AutoValue_RedisLookupDataAdapter_Config.Builder();
}
/* @Override
public Optional<Multimap<String, String>> validate() {
final ArrayListMultimap<String, String> errors = ArrayListMultimap.create();
if (redisPort() < 1 || redisPort() > 65535) {
errors.put("redis_port", "Value cannot neither be smaller than 1 nor greater than 65535");
}
if (redisDB() < 0) {
errors.put("redis_database", "Value cannot be smaller than 0");
}
return errors.isEmpty() ? Optional.empty() : Optional.of(errors);
}
*/
@AutoValue.Builder
public abstract static class Builder {
@JsonCreator
public static Builder create() {
return Config.builder();
}
@JsonProperty(TYPE_FIELD)
public abstract Builder type(String type);
@JsonProperty("redis_host")
public abstract Builder redisHost(String redisHost);
@JsonProperty("redis_port")
public abstract Builder redisPort(int redisPort);
@JsonProperty("redis_database")
public abstract Builder redisDB(int redisDB);
/*@JsonProperty("redis_username")
public abstract Builder redisUsername(String redisUsername);
@JsonProperty("redis_password")
public abstract Builder redisPassword(String redisPassword);*/
public abstract Config build();
}
}
}

View File

@ -0,0 +1,70 @@
/*
* Copyright (C) 2020 Graylog, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
* as published by MongoDB, Inc.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Server Side Public License for more details.
*
* You should have received a copy of the Server Side Public License
* along with this program. If not, see
* <http://www.mongodb.com/licensing/server-side-public-license>.
*/
package in.nosd.redis.functions;
import com.google.common.collect.ForwardingMap;
import com.google.common.collect.ImmutableMap;
import java.util.Map;
public class GenericLookupResult extends ForwardingMap<String, Object> {
public static final String RESULTS_KEY = "threat_indicated";
private final ImmutableMap<String, Object> results;
public static final GenericLookupResult FALSE = new FalseGenericLookupResult();
public static final GenericLookupResult TRUE = new TrueGenericLookupResult();
private GenericLookupResult(ImmutableMap<String, Object> fields) {
this.results = fields;
}
public Map<String, Object> getResults() {
return results;
}
public boolean isMatch() {
return ((boolean) getResults().get(RESULTS_KEY));
}
@Override
protected Map<String, Object> delegate() {
return getResults();
}
private static class FalseGenericLookupResult extends GenericLookupResult {
private static final ImmutableMap<String, Object> FALSE = ImmutableMap.<String, Object>builder()
.put(RESULTS_KEY, false)
.build();
private FalseGenericLookupResult() {
super(FALSE);
}
}
private static class TrueGenericLookupResult extends GenericLookupResult {
private static final ImmutableMap<String, Object> TRUE = ImmutableMap.<String, Object>builder()
.put(RESULTS_KEY, true)
.build();
private TrueGenericLookupResult() {
super(TRUE);
}
}
}

View File

@ -0,0 +1,22 @@
/*
* Copyright (C) 2020 Graylog, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
* as published by MongoDB, Inc.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Server Side Public License for more details.
*
* You should have received a copy of the Server Side Public License
* along with this program. If not, see
* <http://www.mongodb.com/licensing/server-side-public-license>.
*/
package in.nosd.redis.functions;
import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction;
public abstract class LookupTableFunction<R> extends AbstractFunction<R> {
}

View File

@ -0,0 +1,85 @@
/*
* Copyright (C) 2020 Graylog, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
* as published by MongoDB, Inc.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Server Side Public License for more details.
*
* You should have received a copy of the Server Side Public License
* along with this program. If not, see
* <http://www.mongodb.com/licensing/server-side-public-license>.
*/
package in.nosd.redis.functions;
import com.google.common.base.Strings;
import in.nosd.redis.functions.GenericLookupResult;
import in.nosd.redis.functions.LookupTableFunction;
import org.graylog.plugins.pipelineprocessor.EvaluationContext;
import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs;
import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor;
import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor;
import org.graylog2.lookup.LookupTableService;
import org.graylog2.plugin.lookup.LookupResult;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.inject.Inject;
public class RedisLookupPluginFunction extends LookupTableFunction<GenericLookupResult> {
private static final Logger LOG = LoggerFactory.getLogger(RedisLookupPluginFunction.class);
public static final String NAME = "redis_lookup";
private static final String VALUE = "value";
private static final String LOOKUP_TABLE_NAME = "redis-lookup";
private final ParameterDescriptor<String, String> keyParam = ParameterDescriptor.string(VALUE).description("The key to look up.").build();
private final LookupTableService.Function lookupFunction;
@Inject
public RedisLookupPluginFunction(final LookupTableService lookupTableService) {
this.lookupFunction = lookupTableService.newBuilder().lookupTable(LOOKUP_TABLE_NAME).build();
}
@Override
public GenericLookupResult evaluate(FunctionArgs args, EvaluationContext context) {
String key = keyParam.required(args, context);
if (key == null) {
LOG.error("NULL parameter passed to Redis lookup.");
return null;
}
LOG.debug("Running Redis lookup for key [{}].", key);
final LookupResult lookupResult = this.lookupFunction.lookup(key.trim());
if (lookupResult != null && !lookupResult.isEmpty()) {
// If not a String, then fall through to false at the end of the method.
final Object value = lookupResult.singleValue();
if (value instanceof String) {
return !Strings.isNullOrEmpty((String) value) ? GenericLookupResult.TRUE : GenericLookupResult.FALSE;
}
}
return GenericLookupResult.FALSE;
}
@Override
public FunctionDescriptor<GenericLookupResult> descriptor() {
return FunctionDescriptor.<GenericLookupResult>builder()
.name(NAME)
.description("Match a key into Redis instance and return value")
.params(keyParam)
.returnType(GenericLookupResult.class)
.build();
}
}