Modify ulog configuration to use pgsql

pgsql-schema.sql

@@ -0,0 +1,174 @@
+-- vi: et ai ts=2
+-- 
+-- Warning: postgresql >= 8.2 is required for the 'DROP .. IF EXISTS'
+-- Warning: this script DESTROYS EVERYTHING !
+-- 
+-- NOTE : - we could / should use types cidr / inet / macaddr for IP ? (see http://www.postgresql.org/docs/8.2/static/datatype-net-types.html)
+--        - ON UPDATE is not supported ?
+--        - type 'integer' is used (we have to check for overflows ..)
+--        - type 'datetime' has been replaced by 'timestamp'
+
+
+DROP TABLE IF EXISTS ulog2_ct CASCADE;
+
+-- 
+-- conntrack
+-- 
+CREATE TABLE ulog2_ct (
+  ct_id bigint PRIMARY KEY UNIQUE NOT NULL,
+  oob_family smallint default NULL,
+  orig_ip_saddr_str inet default NULL,
+  orig_ip_daddr_str inet default NULL,
+  orig_ip_protocol smallint default NULL,
+  orig_l4_sport integer default NULL,
+  orig_l4_dport integer default NULL,
+  orig_raw_pktlen bigint default 0,
+  orig_raw_pktcount bigint default 0,
+  reply_ip_saddr_str inet default NULL,
+  reply_ip_daddr_str inet default NULL,
+  reply_ip_protocol smallint default NULL,
+  reply_l4_sport integer default NULL,
+  reply_l4_dport integer default NULL,
+  reply_raw_pktlen bigint default 0,
+  reply_raw_pktcount bigint default 0,
+  icmp_code smallint default NULL,
+  icmp_type smallint default NULL,
+  ct_mark bigint default 0,
+  flow_start_sec bigint default 0,
+  flow_start_usec bigint default 0,
+  flow_end_sec bigint default 0,
+  flow_end_usec bigint default 0,
+  ct_event smallint default 0
+);
+
+-- 
+-- Additional INDEX
+-- 
+
+-- CREATE INDEX ulog2_ct_oob_family ON ulog2_ct(oob_family);
+-- CREATE INDEX ulog2_ct_orig_ip_saddr ON ulog2_ct(orig_ip_saddr_str);
+-- CREATE INDEX ulog2_ct_orig_ip_daddr ON ulog2_ct(orig_ip_daddr_str);
+-- CREATE INDEX ulog2_ct_reply_ip_saddr ON ulog2_ct(reply_ip_saddr_str);
+-- CREATE INDEX ulog2_ct_reply_ip_daddr ON ulog2_ct(reply_ip_daddr_str);
+-- CREATE INDEX ulog2_ct_orig_l4_sport ON ulog2_ct(orig_l4_sport);
+-- CREATE INDEX ulog2_ct_orig_l4_dport ON ulog2_ct(orig_l4_dport);
+-- CREATE INDEX ulog2_ct_reply_l4_sport ON ulog2_ct(reply_l4_sport);
+-- CREATE INDEX ulog2_ct_reply_l4_dport ON ulog2_ct(reply_l4_dport);
+-- CREATE INDEX ulog2_ct_event ON ulog2_ct(ct_event);
+
+-- 
+-- Procedures
+-- 
+
+CREATE OR REPLACE FUNCTION INSERT_CT(
+                IN _ct_id integer,
+                IN _oob_family integer,
+                IN _orig_ip_saddr inet,
+                IN _orig_ip_daddr inet,
+                IN _orig_ip_protocol integer,
+                IN _orig_l4_sport integer,
+                IN _orig_l4_dport integer,
+                IN _orig_raw_pktlen bigint,
+                IN _orig_raw_pktcount bigint,
+                IN _reply_ip_saddr inet,
+                IN _reply_ip_daddr inet,
+                IN _reply_ip_protocol integer,
+                IN _reply_l4_sport integer,
+                IN _reply_l4_dport integer,
+                IN _reply_raw_pktlen bigint,
+                IN _reply_raw_pktcount bigint,
+                IN _icmp_code integer,
+                IN _icmp_type integer,
+                IN _ct_mark bigint,
+                IN _flow_start_sec bigint,
+                IN _flow_start_usec bigint,
+                IN _flow_end_sec bigint,
+                IN _flow_end_usec bigint,
+                IN _ct_event integer
+        )
+RETURNS bigint AS $$
+        INSERT INTO ulog2_ct (ct_id, oob_family, orig_ip_saddr_str, orig_ip_daddr_str, orig_ip_protocol,
+                        orig_l4_sport, orig_l4_dport, orig_raw_pktlen, orig_raw_pktcount,
+                        reply_ip_saddr_str, reply_ip_daddr_str, reply_ip_protocol,
+                        reply_l4_sport, reply_l4_dport, reply_raw_pktlen, reply_raw_pktcount,
+                        icmp_code, icmp_type, ct_mark, 
+                        flow_start_sec, flow_start_usec,
+                        flow_end_sec, flow_end_usec, ct_event)
+                VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24);
+        RETURN _ct_id;
+$$ LANGUAGE SQL SECURITY INVOKER;
+
+CREATE OR REPLACE FUNCTION INSERT_OR_REPLACE_CT(
+                IN _ct_id integer,
+                IN _oob_family integer,
+                IN _orig_ip_saddr inet,
+                IN _orig_ip_daddr inet,
+                IN _orig_ip_protocol integer,
+                IN _orig_l4_sport integer,
+                IN _orig_l4_dport integer,
+                IN _orig_raw_pktlen bigint,
+                IN _orig_raw_pktcount bigint,
+                IN _reply_ip_saddr inet,
+                IN _reply_ip_daddr inet,
+                IN _reply_ip_protocol integer,
+                IN _reply_l4_sport integer,
+                IN _reply_l4_dport integer,
+                IN _reply_raw_pktlen bigint,
+                IN _reply_raw_pktcount bigint,
+                IN _icmp_code integer,
+                IN _icmp_type integer,
+                IN _ct_mark bigint,
+                IN _flow_start_sec bigint,
+                IN _flow_start_usec bigint,
+                IN _flow_end_sec bigint,
+                IN _flow_end_usec bigint,
+                IN _ct_event integer
+        )
+RETURNS bigint AS $$
+DECLARE
+        _id bigint;
+BEGIN
+        IF (_ct_event = 4) THEN
+          if (_orig_ip_protocol = 1) THEN
+            UPDATE ulog2_ct SET (orig_raw_pktlen, orig_raw_pktcount,
+                reply_raw_pktlen, reply_raw_pktcount,
+                ct_mark, flow_end_sec, flow_end_usec, ct_event)
+                = ($8,$9,$15,$16,$19,$22,$23,$24)
+            WHERE ct_id=$1 AND oob_family=$2 AND orig_ip_saddr_str = $3
+                AND orig_ip_daddr_str = $4 AND orig_ip_protocol = $5
+                AND reply_ip_saddr_str = $10 AND reply_ip_daddr_str = $11
+                AND reply_ip_protocol = $12
+                AND icmp_code = $17 AND icmp_type = $18 
+                AND ct_event < 4;
+          ELSE
+            UPDATE ulog2_ct SET (orig_raw_pktlen, orig_raw_pktcount,
+                reply_raw_pktlen, reply_raw_pktcount,
+                ct_mark, flow_end_sec, flow_end_usec, ct_event)
+                = ($8,$9,$15,$16,$19,$22,$23,$24)
+            WHERE ct_id=$1 AND oob_family=$2 AND orig_ip_saddr_str = $3
+                AND orig_ip_daddr_str = $4 AND orig_ip_protocol = $5
+                AND orig_l4_sport = $6 AND orig_l4_dport = $7
+                AND reply_ip_saddr_str = $10 AND reply_ip_daddr_str = $11
+                AND reply_ip_protocol = $12 AND reply_l4_sport = $13
+                AND reply_l4_dport = $14 
+                AND ct_event < 4;
+          END IF;
+        ELSE
+          _id := INSERT_CT($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24);
+        END IF;
+        RETURN _id;
+END
+$$ LANGUAGE plpgsql SECURITY INVOKER;
+
+
+CREATE OR REPLACE FUNCTION DELETE_CT_FLOW(
+                IN _ct_packet_id bigint
+        )
+RETURNS void AS $$
+  -- remember : table with most constraints first
+  DELETE FROM ulog2_ct WHERE ulog2_ct._ct_id = $1;
+$$ LANGUAGE SQL SECURITY INVOKER;
+
+
+-- Created by Pierre Chifflier <chifflier AT inl DOT fr>
+-- Adapted by Thomas Chevalier <contact AT tchevalier DOT fr>

ulogd.conf

@@ -2,14 +2,8 @@
 # https://connect.ed-diamond.com/GNU-Linux-Magazine/glmfhs-041/ulogd2-journalisation-avancee-avec-netfilter
 
 [global]
-######################################################################
-# GLOBAL OPTIONS
-######################################################################
-
-
 # logfile for status messages
 logfile="syslog"
-
 # loglevel: debug(1), info(3), notice(5), error(7) or fatal(8) (default 5)
 loglevel=3
 
@@ -58,44 +52,48 @@ stack=log1:NFLOG,base1:BASE,ifi1:IFINDEX,ip2str1:IP2STR,print1:PRINTPKT,emu1:LOG
 # Packet logging
 stack=log2:NFLOG,base1:BASE,ifi1:IFINDEX,ip2str1:IP2STR,mac2str1:HWHDR,print1:PRINTPKT,json1:JSON
 
+# Conntrack logging
+stack=ct1:NFCT,ip2str1:IP2STR,printflow1:PRINTFLOW,json2:JSON
+stack=ct1:NFCT,ip2bin1:IP2BIN,pgsql1:PGSQL
+
+
 # Logging of system packet through NFLOG
 [log1]
 # netlink multicast group (the same as the iptables --nflog-group param)
 # Group O is used by the kernel to log connection tracking invalid message
 group=0
-#netlink_socket_buffer_size=217088
+
-#netlink_socket_buffer_maxsize=1085440
+# General packet logging
-# set number of packet to queue inside kernel
-#netlink_qthreshold=1
-# set the delay before flushing packet in the queue inside kernel (in 10ms)
-#netlink_qtimeout=100
-
-# packet logging through NFLOG for group 1
 [log2]
-# netlink multicast group (the same as the iptables --nflog-group param)
+# Group has to be different from the one use in log1
-group=1 # Group has to be different from the one use in log1
+group=1
-#netlink_socket_buffer_size=217088
+
-#netlink_socket_buffer_maxsize=1085440
+[ct1]
-# If your kernel is older than 2.6.29 and if a NFLOG input plugin with
+# NEW     = 1
-# group 0 is not used by any stack, you need to have at least one NFLOG
+# UPDATE  = 2
-# input plugin with bind set to 1. If you don't do that you may not
+# DESTROY = 4
-# receive any message from the kernel.
+event_mask=0x0000005
-#bind=1
+# If hash_enable=1 (the default), the kernel will automatically
+# match NEW and DESTROY events and only report DESTROY events.
+hash_enable=0
+# reliable=1 # enable reliable flow-based logging (may drop packets)
 
 [emu1]
-file="/var/log/ulog/syslogemu.log"
+file="/var/log/ulog/kernel.log"
 sync=1
 
 [json1]
 sync=1
 file="/var/log/ulog/ulogd.json"
-#timestamp=0
+
-# device name to be used in JSON message
+[json2]
-#device="My awesome Netfilter firewall"
+sync=1
-# If boolean_label is set to 1 then the numeric_label put on packet
+file="/var/log/ulog/ct.json"
-# by the input plugin is coding the action on packet: if 0, then
+
-# packet has been blocked and if non null it has been accepted.
+[pgsql1]
-#boolean_label=1
+db="ulog2"
-# Uncomment the following line to use JSON v1 event format that
+host="replace-with-cloud-sql-ip-address"
-# can provide better compatility with some JSON file reader.
+user="ulog"
-#eventv1=1
+table="ulog2_ct"
+pass="replace-with-database-password"
+procedure="INSERT_OR_REPLACE_CT"