openldap-log-parser.go

@@ -18,7 +18,7 @@ const (
 	// FIXME: Not very strict
 	IPv6RE            = `(?:(?:[0-9a-fA-F]{1,4}\:){7})[0-9a-fA-F]{1,4}`
 	HostRE            = `([0-9A-Za-z\-\_\.]*)`
-	ProcessRE         = `(slapd\[[0-9]{1,7}\])(?:\:)?`
+	ProcessRE         = `(slapd\[[0-9]{1,5}\])`
 	// group[4]
 	ConnIdRE          = `conn=([0-9]{4,10})`
 	ConnFdRE          = `(?:fd=([0-9]{1,10}))?`
@@ -43,14 +43,10 @@ const (
 	UnbindRE          = `(UNBIND)?`
 	// group[40]
 	ConnClosedRE      = `(closed)?(?: \(connection lost\))?`
-	// group[41]
-	AddDnRE           = `(?:ADD dn="(.*)")?`
-	// group[42]
-	DelDnRE           = `(?:DEL dn="(.*)")?`
 	
 	LogLineRE = SyslogPri + TimeRE + ` ` + HostRE + ` ` + ProcessRE + ` ` + ConnIdRE + ` ` + ConnFdRE + OperationIdRE + ` ` +
 				AcceptRE + STARTTLSRE + BindMethodRE + BindMechRE + ResultRE + SearchBaseRE + SearchAttrRE + SearchResultRE + ModDnRE + ModAttrRE + 
-				PassModRE + UnbindRE + ConnClosedRE + AddDnRE + DelDnRE
+				PassModRE + UnbindRE + ConnClosedRE
 )
 
 type (
@@ -78,8 +74,6 @@ type (
 		SSF            string     `json:"ssf"`
 		ModDN          string     `json:"mod_dn"`
 		ModAttr        string     `json:"mod_attr"`
-		AddDN          string     `json:"add_dn"`
-		DelDN          string     `json:"del_dn"`
 		PassModDN      string     `json:"passmod_dn"`
 		Result         bool
 		ResTag         string     `json:"result_tag"`
@@ -190,8 +184,6 @@ func (o *OpenldapLog) Parse(text []byte) (LogFormat, error) {
 		ModDN:             string(group[36]),
 		ModAttr:           string(group[37]),
 		PassModDN:         string(group[38]),
-		AddDN:             string(group[41]),
-		DelDN:             string(group[42]),
 	}
 
 	// Now handle Operation Type
@@ -211,10 +203,6 @@ func (o *OpenldapLog) Parse(text []byte) (LogFormat, error) {
 		logFormat.OpType = "unbind"
 	} else if len(group[40]) > 0 {
 		logFormat.OpType = "close"
-	} else if len(group[41]) > 0 {
-		logFormat.OpType = "add"
-	} else if len(group[42]) > 0 {
-		logFormat.OpType = "del"
 	}
 
 	return logFormat, nil

openldap-log-parser/cmd/root.go

@@ -57,8 +57,6 @@ type (
 		SSF            string     `json:"ssf,omitempty"`
 		ModDN          string     `json:"mod_dn,omitempty"`
 		ModAttr        string     `json:"mod_attr,omitempty"`
-		AddDN          string     `json:"add_dn,omitempty"`
-		DelDN          string     `json:"del_dn,omitempty"`
 		PassModDN      string     `json:"passmod_dn,omitempty"`
 		ResTag         string     `json:"result_tag,omitempty"`
 		ResOid         string     `json:"result_oid,omitempty"`
@@ -102,8 +100,6 @@ type (
 		StartTLS       bool       `json:"starttls,omitempty"`
 		ModDN          string     `json:"mod_dn,omitempty"`
 		ModAttr        string     `json:"mod_attr,omitempty"`
-		AddDN          string     `json:"add_dn,omitempty"`
-		DelDN          string     `json:"del_dn,omitempty"`
 		PassModDN      string     `json:"passmod_dn,omitempty"`
 		ResTag         string     `json:"result_tag,omitempty"`
 		ResOid         string     `json:"result_oid,omitempty"`
@@ -129,7 +125,7 @@ var (
 	File   os.File
 	Writer *bufio.Writer
 
-	Version = "0.6.13"
+	Version = "0.6.9"
 
 	BuildInfo = promauto.NewGaugeVec(prometheus.GaugeOpts{
 		Name: "openldaplogparser_build_info",
@@ -155,43 +151,35 @@ var (
 		Name: "openldaplogparser_client_count",
 		Help: "Number of connected clients",
 	})
-	AcceptCnt = promauto.NewCounterVec(prometheus.CounterOpts{
+  AcceptCnt = promauto.NewCounterVec(prometheus.CounterOpts{
 		Name: "openldaplogparser_accept_count",
 		Help: "Number of ACCEPT commands executed",
 	}, []string{"host"})
-	BindCnt = promauto.NewCounterVec(prometheus.CounterOpts{
+  BindCnt = promauto.NewCounterVec(prometheus.CounterOpts{
 		Name: "openldaplogparser_bind_count",
 		Help: "Number of BIND commands executed",
 	}, []string{"host"})
-	SearchCnt = promauto.NewCounterVec(prometheus.CounterOpts{
+  SearchCnt = promauto.NewCounterVec(prometheus.CounterOpts{
 		Name: "openldaplogparser_search_count",
 		Help: "Number of SRCH commands executed",
 	}, []string{"host"})
-	ModCnt = promauto.NewCounterVec(prometheus.CounterOpts{
+  ModCnt = promauto.NewCounterVec(prometheus.CounterOpts{
 		Name: "openldaplogparser_mod_count",
 		Help: "Number of MOD commands executed",
 	}, []string{"host"})
-	AddCnt = promauto.NewCounterVec(prometheus.CounterOpts{
+  PassModCnt = promauto.NewCounterVec(prometheus.CounterOpts{
-	  Name: "openldaplogparser_add_count",
-	  Help: "Number of ADD commands executed",
-	}, []string{"host"})
-	DelCnt = promauto.NewCounterVec(prometheus.CounterOpts{
-		Name: "openldaplogparser_del_count",
-		Help: "Number of DEL commands executed",
-	}, []string{"host"})
-	PassModCnt = promauto.NewCounterVec(prometheus.CounterOpts{
 		Name: "openldaplogparser_passmod_count",
 		Help: "Number of PASSMOD commands executed",
 	}, []string{"host"})
-	UnbindCnt = promauto.NewCounterVec(prometheus.CounterOpts{
+  UnbindCnt = promauto.NewCounterVec(prometheus.CounterOpts{
 		Name: "openldaplogparser_unbind_count",
 		Help: "Number of UNBIND commands executed",
 	}, []string{"host"})
-	CloseCnt = promauto.NewCounterVec(prometheus.CounterOpts{
+  CloseCnt = promauto.NewCounterVec(prometheus.CounterOpts{
 		Name: "openldaplogparser_close_count",
 		Help: "Number of closed connections",
 	}, []string{"host"})
-	StartTLSCnt = promauto.NewCounterVec(prometheus.CounterOpts{
+  StartTLSCnt = promauto.NewCounterVec(prometheus.CounterOpts{
 		Name: "openldaplogparser_starttlscount",
 		Help: "Number of STARTTLS commands executed",
 	}, []string{"host"})
@@ -285,22 +273,6 @@ func OlcToFlat(olc *OpenLdapConnection) []OpenLdapConnectionFlat {
 				olcf[i].ResQTime = olc.Operations[i].ResQTime
 				olcf[i].ResETime = olc.Operations[i].ResETime
 				olcf[i].ResText = olc.Operations[i].ResText
-			case "add":
-				olcf[i].AddDN = olc.Operations[i].AddDN
-				olcf[i].ResTag = olc.Operations[i].ResTag
-				olcf[i].ResOid = olc.Operations[i].ResOid
-				olcf[i].ResErr = olc.Operations[i].ResErr
-				olcf[i].ResQTime = olc.Operations[i].ResQTime
-				olcf[i].ResETime = olc.Operations[i].ResETime
-				olcf[i].ResText = olc.Operations[i].ResText
-			case "del":
-				olcf[i].DelDN = olc.Operations[i].DelDN
-				olcf[i].ResTag = olc.Operations[i].ResTag
-				olcf[i].ResOid = olc.Operations[i].ResOid
-				olcf[i].ResErr = olc.Operations[i].ResErr
-				olcf[i].ResQTime = olc.Operations[i].ResQTime
-				olcf[i].ResETime = olc.Operations[i].ResETime
-				olcf[i].ResText = olc.Operations[i].ResText
 			case "passmod":
 				olcf[i].PassModDN = olc.Operations[i].PassModDN
 				olcf[i].ResTag = olc.Operations[i].ResTag
@@ -349,38 +321,29 @@ func writeOut(msg string, filename string) error {
 	return nil
 }
 
-func cleanMQueue(mqueue map[string]*OpenLdapConnection, mqMtx *sync.Mutex, age time.Duration) {
-    var ok bool
-
-	log.Printf("Start cleaning queue task: %d items in queue", len(mqueue))
-
-	// We need lock here
-	mqMtx.Lock()
-	for uid, ldcon := range mqueue {
-		ok = false
-		// Check if a close operation exist
-		for _, op := range ldcon.Operations {
-			if op.OpType == "close" {
-				if op.Time.Add(age).Before(time.Now()) {
-					ok = true
-				}
-			}
-		}
-		if ok == true {
-			// We already in RW lock
-			delete(mqueue, uid)
-		}
-	}
-	mqMtx.Unlock()
-	log.Printf("Finished cleaning queue task: %d items in queue", len(mqueue))
-}
-
-
-
 // Every 24H, remove connections closed more than 24 hours ago
 func periodicallyCleanMQueue(mqueue map[string]*OpenLdapConnection, mqMtx *sync.Mutex) {
+	var ok bool
+
 	for range time.Tick(time.Hour * 24) {
-		cleanMQueue(mqueue, mqMtx, 24 * time.Hour)
+		// Do we need read lock?
+		for _, ldcon := range mqueue {
+			ok = false
+			// Check if a close operation exist
+			for _, op := range ldcon.Operations {
+				if op.OpType == "close" {
+					if op.Time.Add(time.Hour * 1 * 24).Before(time.Now()) {
+						ok = true
+					}
+				}
+			}
+			if ok == true {
+				mqMtx.Lock()
+				//delete(mqueue, ldcon.ConnId)
+				delete(mqueue, fmt.Sprintf("%s:%d", ldcon.Hostname, ldcon.ConnId))
+				mqMtx.Unlock()
+			}
+		}
 	}
 }
 
@@ -765,82 +728,6 @@ func parseStoreAndWrite(input []byte, mq map[string]*OpenLdapConnection, mqMtx *
 		return nil
 	}
   
-	/*
-	 * 2022-07-18T14:35:17.381223+02:00 ldap.domain.org slapd slapd[82581] conn=16113 op=3 ADD dn="cn=coincoin,dc=domain,dc=org"
-	 */
-	if logFormat.AddDN != "" {
-		op := &Operation{
-			Time:           logFormat.Time,
-			OpType:         logFormat.OpType,
-			OpId:           &logFormat.OpId,
-			AddDN:          logFormat.AddDN,
-		}
-		mqMtx.Lock()
-		olc, ok := mq[fmt.Sprintf("%s:%d", logFormat.Hostname, logFormat.ConnId)]
-		if false == ok {
-			if false == gDispUnkConn {
-				mqMtx.Unlock()
-				return nil
-			} else {
-				// Create connection
-				olc = &OpenLdapConnection{
-					Time:           logFormat.Time,
-					Hostname:       logFormat.Hostname,
-					Process:        logFormat.Process,
-					ConnId:         logFormat.ConnId,
-					ConnFd:         logFormat.ConnFd,
-					ClientIp:       logFormat.ClientIp,
-					ClientPort:     logFormat.ClientPort,
-					ServerIp:       logFormat.ServerIp,
-					ServerPort:     logFormat.ServerPort,
-				}
-				mq[fmt.Sprintf("%s:%d", logFormat.Hostname, logFormat.ConnId)] = olc
-			}
-		}
-		olc.Operations = append(olc.Operations, op)
-		mqMtx.Unlock()
-		AddCnt.WithLabelValues(olc.Hostname).Inc()
-		return nil
-	}
-	
-	/*
-	 * 2022-07-18T14:35:17.381223+02:00 ldap.domain.org slapd slapd[82581] conn=16113 op=3 DEL dn="cn=coincoin,dc=domain,dc=org"
-	 */
-	if logFormat.DelDN != "" {
-		op := &Operation{
-			Time:           logFormat.Time,
-			OpType:         logFormat.OpType,
-			OpId:           &logFormat.OpId,
-			DelDN:          logFormat.DelDN,
-		}
-		mqMtx.Lock()
-		olc, ok := mq[fmt.Sprintf("%s:%d", logFormat.Hostname, logFormat.ConnId)]
-		if false == ok {
-			if false == gDispUnkConn {
-				mqMtx.Unlock()
-				return nil
-			} else {
-				// Create connection
-				olc = &OpenLdapConnection{
-					Time:           logFormat.Time,
-					Hostname:       logFormat.Hostname,
-					Process:        logFormat.Process,
-					ConnId:         logFormat.ConnId,
-					ConnFd:         logFormat.ConnFd,
-					ClientIp:       logFormat.ClientIp,
-					ClientPort:     logFormat.ClientPort,
-					ServerIp:       logFormat.ServerIp,
-					ServerPort:     logFormat.ServerPort,
-				}
-				mq[fmt.Sprintf("%s:%d", logFormat.Hostname, logFormat.ConnId)] = olc
-			}
-		}
-		olc.Operations = append(olc.Operations, op)
-		mqMtx.Unlock()
-		DelCnt.WithLabelValues(olc.Hostname).Inc()
-		return nil
-	}
-  
 	/*
 		2022-07-18T11:13:17.521717+02:00 ldap.domain.org slapd[82581] conn=16113 op=4 PASSMOD id="cn=pika,ou=users,dc=domain,dc=org" new
 	*/
@@ -1086,17 +973,6 @@ func processLogs(cmd *cobra.Command, args []string) {
 	// Cleaner thread
 	go periodicallyCleanMQueue(mQueue, &mqMtx)
 
-    // On demand Mqueue cleaning... For debug, dont try this at home, kids!
-/*	sig2 := make(chan os.Signal)
-	signal.Notify(sig2, syscall.SIGUSR2)
-	go func() {
-		for {
-			<-sig2
-			cleanMQueue(mQueue, &mqMtx, 5 * time.Minute)
-		}
-	}()
-*/
-
 	// Initialize Stdin input...
 	if true == strings.EqualFold(gSyslogListenAddress, "do-not-listen") {
 		useStdin = true

rsyslog/openldap-log-parser.conf

@@ -1,165 +0,0 @@
-module(load="imfile")       # lecture slapd.log.json
-module(load="mmjsonparse")  # parsing slapd.log.json
-
-# Template de mise en forme JSON
-template(name="sendJsonToGrayLogTemplate"
-    type="list" option.json="on") {
-    constant(value="{ ")
-    constant(value="\"facility\":\"local4\", ")
-    constant(value="\"facility_num\":\"20\", ")
-    constant(value="\"level\":\"6\", ")
-    constant(value="\"type\":\"")
-    property(name="programname")
-    constant(value="\", ")
-    # on renomme les proprietes venant de openldap-log-parser
-    constant(value="\"time\":\"")
-    property(name="$!time")
-    constant(value="\", ")
-    constant(value="\"source\":\"")
-    property(name="$!hostname")
-    constant(value="\", ")
-    constant(value="\"process\":\"")
-    property(name="$!process")
-    constant(value="\", ")
-    constant(value="\"client_ip\":\"")
-    property(name="$!client_ip")
-    constant(value="\", ")
-    constant(value="\"client_port\":\"")
-    property(name="$!client_port")
-    constant(value="\", ")
-    constant(value="\"server_ip\":\"")
-    property(name="$!server_ip")
-    constant(value="\", ")
-    constant(value="\"server_port\":\"")
-    property(name="$!server_port")
-    constant(value="\", ")
-    constant(value="\"bind_dn\":\"")
-    property(name="$!bind_dn")
-    constant(value="\", ")
-    constant(value="\"conn_id\":\"")
-    property(name="$!conn_id")
-    constant(value="\", ")
-    constant(value="\"conn_fd\":\"")
-    property(name="$!conn_fd")
-    constant(value="\", ")
-    constant(value="\"op_id\":\"")
-    property(name="$!op_id")
-    constant(value="\", ")
-    constant(value="\"op_type\":\"")
-    property(name="$!op_type")
-    constant(value="\", ")
-    constant(value="\"bind_method\":\"")
-    property(name="$!bind_method")
-    constant(value="\", ")
-    constant(value="\"bind_mech\":\"")
-    property(name="$!bind_mech")
-    constant(value="\", ")
-    constant(value="\"bind_ssf\":\"")
-    property(name="$!bind_ssf")
-    constant(value="\", ")
-    constant(value="\"ssf\":\"")
-    property(name="$!ssf")
-    constant(value="\", ")
-    constant(value="\"starttls\":\"")
-    property(name="$!starttls")
-    constant(value="\", ")
-    constant(value="\"mod_dn\":\"")
-    property(name="$!mod_dn")
-    constant(value="\", ")
-    constant(value="\"mod_attr\":\"")
-    property(name="$!mod_attr")
-    constant(value="\", ")
-    constant(value="\"add_dn\":\"")
-    property(name="$!add_dn")
-    constant(value="\", ")
-    constant(value="\"del_dn\":\"")
-    property(name="$!del_dn")
-    constant(value="\", ")
-    constant(value="\"passmod_dn\":\"")
-    property(name="$!passmod_dn")
-    constant(value="\", ")
-    constant(value="\"res_tag\":\"")
-    property(name="$!result_tag")
-    constant(value="\", ")
-    constant(value="\"res_oid\":\"")
-    property(name="$!result_oid")
-    constant(value="\", ")
-    constant(value="\"res_err\":\"")
-    property(name="$!result_err")
-    constant(value="\", ")
-    constant(value="\"res_qtime\":\"")
-    property(name="$!result_qtime")
-    constant(value="\", ")
-    constant(value="\"res_etime\":\"")
-    property(name="$!result_etime")
-    constant(value="\", ")
-    constant(value="\"res_text\":\"")
-    property(name="$!result_text")
-    constant(value="\", ")
-    constant(value="\"search_base\":\"")
-    property(name="$!search_base")
-    constant(value="\", ")
-    constant(value="\"search_scope\":\"")
-    property(name="$!search_scope")
-    constant(value="\", ")
-    constant(value="\"search_deref\":\"")
-    property(name="$!search_deref")
-    constant(value="\", ")
-    constant(value="\"search_filter\":\"")
-    property(name="$!search_filter")
-    constant(value="\", ")
-    constant(value="\"search_attr\":\"")
-    property(name="$!search_attr")
-    constant(value="\", ")
-    constant(value="\"search_res_tag\":\"")
-    property(name="$!search_res_tag")
-    constant(value="\", ")
-    constant(value="\"search_res_err\":\"")
-    property(name="$!search_res_err")
-    constant(value="\", ")
-    constant(value="\"search_res_qtime\":\"")
-    property(name="$!search_res_qtime")
-    constant(value="\", ")
-    constant(value="\"search_res_etime\":\"")
-    property(name="$!search_res_etime")
-    constant(value="\", ")
-    constant(value="\"search_res_nentries\":\"")
-    property(name="$!search_res_nentries")
-    constant(value="\", ")
-    constant(value="\"search_res_text\":\"")
-    property(name="$!search_res_text")
-    constant(value="\", ")
-    constant(value="\"message\":\"")
-    property(name="$!message")
-    constant(value="\" ")
-    constant(value=" }")
-}
-
-# On envoit les logs ldap vers openldap-log-parser qui tourne en tant que service
-if $programname == 'slapd' then action(
-    type="omfwd"
-    Target="127.0.0.1"
-    Port="6514"
-    Protocol="tcp"
-    template="RSYSLOG_FileFormat")
-
-# Le flux post openldap-log-parser, qu'on relit pour envoyer vers graylog
-input(type="imfile"
-    File="/var/log/slapd.log.json"
-    Tag="openldap-agg"
-    addMetadata="on"
-    ruleset="remoteAllJsonLog"
-)
-
-ruleset(name="remoteAllJsonLog") {
-    action(type="mmjsonparse" cookie="")
-    action(
-        type="omfwd"
-        Target="graylog.example.org"
-        Port="2514"
-        Protocol="tcp"
-        template="sendJsonToGrayLogTemplate"
-    )
-    stop
-}
-