First commit

README.md

@@ -0,0 +1,39 @@
+# Introduction
+This project collect some metrics for TCP. For doing that, I use eBPF.
+
+## Requirements
+For executing and loading the eBPF program and to send data to InfluxDB, you need to install some packages:
+
+```
+```
+
+## Compile eBPF program
+First, you need to dump the vmlinux header file, which contains all definitions codes of your Linux kernel:
+
+```
+sudo bpftool btf dump file /sys/kernel/btf/vmlinux format c > vmlinux.h
+```
+
+After that, you can compile the eBPF code:
+
+```
+$ clang-11 -g -O2 -target bpf -c tp_tcp.c -o tp_tcp.o
+```
+
+Now, I made a C script which can load the eBPF program and attach it:
+
+```
+$ gcc load_bpf.c -o load_bpf -lbpf
+```
+
+And you can execute it, but, you need to have the root privileges:
+
+```
+$ sudo ./load_bpf
+```
+
+
+## InfluxDB
+I use this [project](https://github.com/nigelargriffiths/InfluxDB-C-client) for sending data to InfluxDB.
+
+The documentation of that project is [here](https://www.influxdata.com/blog/influxdb-c-client-library-for-capturing-statistics/)

common.h

@@ -0,0 +1,11 @@
+#ifndef H_COMMON
+#define H_COMMON
+
+struct reset {
+    __u8 saddr[4];
+    __u8 daddr[4];
+    __u16 sport;
+    __u16 dport;
+};
+
+#endif

ic.c

@@ -0,0 +1,325 @@
+/*
+ * Influx C (ic) client for data capture
+ * Developer: Nigel Griffiths.
+ * (C) Copyright 2021 Nigel Griffiths
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the gnu general public license as published by
+    the free software foundation, either version 3 of the license, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but without any warranty; without even the implied warranty of
+    merchantability or fitness for a particular purpose.  see the
+    gnu general public license for more details.
+
+    You should have received a copy of the gnu general public license
+    along with this program.  if not, see <http://www.gnu.org/licenses/>.
+
+    Compile: cc ic.c -g -O3 -o ic
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <ctype.h>
+#include <math.h>
+#include <string.h>
+#include <sys/errno.h>
+#include <sys/socket.h>
+#include <arpa/inet.h>
+#include <netdb.h>
+
+#define DEBUG   if(debug)
+#define MEGABYTE ( 1024 * 1024 ) /* USed as the default buffer sizes */
+
+int debug = 0; /* 0=off, 1=on basic, 2=trace like output */
+
+char influx_hostname[1024 + 1] = { 0 };/* details of the influxdb server or telegraf */
+char influx_ip[16 + 1] = { 0 };
+long influx_port = 0;
+
+char influx_database[256+1];		/* the influxdb database  */
+char influx_username[64+1];		/* optional for influxdb access */
+char influx_password[64+1];		/* optional for influxdb access */
+
+char *output; /* all the stats must fit in this buffer */
+long output_size = 0;
+long output_char = 0;
+
+char *influx_tags; /* saved tags for every influxdb line protocol mesurement */
+
+int subended = 0;		/* stop ic_subend and ic_measureend both enig the measure */
+int first_sub = 0;		/* need to remove the ic_measure measure before adding ic_sub measure */
+char saved_section[64];
+char saved_sub[64];
+
+int sockfd;			/* file desciptor for socket connection */
+
+void error(char *buf)
+{
+    fprintf(stderr, "error: \"%s\" errno=%d meaning=\"%s\"\n", buf, errno, strerror(errno));
+    close(sockfd);
+    sleep(2);			/* this can help the socket close cleanly at the remote end */
+    exit(1);
+}
+
+void ic_debug(int level)
+{
+	debug = level;
+}
+
+/* ic_tags() argument is the measurement tags for influddb */
+/* example: "host=vm1234"   note:the comma & hostname of the virtual machine sending the data */
+/* complex: "host=lpar42,serialnum=987654,arch=power9" note:the comma separated list */
+void ic_tags(char *t)	
+{
+    DEBUG fprintf(stderr,"ic_tags(%s)\n",t);
+    if( influx_tags == (char *) 0) {
+        if( (influx_tags = (char *)malloc(MEGABYTE)) == (char *)-1)
+           error("failed to malloc() tags buffer");
+    }
+
+    strncpy(influx_tags,t,256);
+}
+
+void ic_influx_database(char *host, long port, char *db) /* note: converts influxdb hostname to ip address */
+{
+	struct hostent *he;
+	char errorbuf[1024 +1 ];
+
+	influx_port = port;
+	strncpy(influx_database,db,256);
+
+	if(host[0] <= '0' && host[0] <='9') { 
+		DEBUG fprintf(stderr,"ic_influx(ipaddr=%s,port=%ld,database=%s))\n",host,port,db);
+		strncpy(influx_ip,host,16);
+	} else {
+		DEBUG fprintf(stderr,"ic_influx_by_hostname(host=%s,port=%ld,database=%s))\n",host,port,db);
+		strncpy(influx_hostname,host,1024);
+		if (isalpha(host[0])) {
+
+		    he = gethostbyname(host);
+		    if (he == NULL) {
+			sprintf(errorbuf, "influx host=%s to ip address convertion failed gethostbyname(), bailing out\n", host);
+			error(errorbuf);
+		    }
+		    /* this could return multiple ip addresses but we assume its the first one */
+		    if (he->h_addr_list[0] != NULL) {
+			strcpy(influx_ip, inet_ntoa(*(struct in_addr *) (he->h_addr_list[0])));
+			DEBUG fprintf(stderr,"ic_influx_by_hostname hostname=%s converted to ip address %s))\n",host,influx_ip);
+		    } else {
+			sprintf(errorbuf, "influx host=%s to ip address convertion failed (empty list), bailing out\n", host);
+			error(errorbuf);
+		    }
+		} else {
+		    strcpy( influx_ip, host); /* perhaps the hostname is actually an ip address */
+		}
+        }
+}
+
+void ic_influx_userpw(char *user, char *pw)
+{
+	DEBUG fprintf(stderr,"ic_influx_userpw(username=%s,pssword=%s))\n",user,pw);
+	strncpy(influx_username,user,64);
+	strncpy(influx_password,pw,64);
+}
+
+int create_socket() 		/* returns 1 for error and 0 for ok */
+{
+    int i;
+    static char buffer[4096];
+    static struct sockaddr_in serv_addr;
+
+    if(debug) DEBUG fprintf(stderr, "socket: trying to connect to \"%s\":%ld\n", influx_ip, influx_port);
+    if ((sockfd = socket(AF_INET, SOCK_STREAM, 0)) < 0) {
+	error("socket() call failed");
+	return 0;
+    }
+
+    serv_addr.sin_family = AF_INET;
+    serv_addr.sin_addr.s_addr = inet_addr(influx_ip);
+    serv_addr.sin_port = htons(influx_port);
+
+    /* connect tot he socket offered by the web server */
+    if (connect(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) {
+	DEBUG fprintf(stderr, " connect() call failed errno=%d", errno);
+	return 0;
+    }
+    return 1;
+}
+
+void ic_check(long adding) /* Check the buffer space */
+{
+    if(output == (char *)0) {			/* First time create the buffer *
+	if( (output = (char *)malloc(MEGABYTE)) == (char *)-1)
+	    error("failed to malloc() output buffer");
+    }
+    if(output_char + (2*adding) > output_size) /* When near the end of the output buffer, extend it*/
+	if( (output = (char *)realloc(output, output_size + MEGABYTE)) == (char *)-1)
+	    error("failed to realloc() output buffer");
+    }
+}
+
+void remove_ending_comma_if_any()
+{
+    if (output[output_char - 1] == ',') {
+	output[output_char - 1] = 0;	/* remove the char */
+	output_char--;
+    }
+}
+
+void ic_measure(char *section)
+{
+    ic_check( strlen(section) + strlen(influx_tags) + 3);
+
+    output_char += sprintf(&output[output_char], "%s,%s ", section, influx_tags);
+    strcpy(saved_section, section);
+    first_sub = 1;
+    subended = 0;
+    DEBUG fprintf(stderr, "ic_measure(\"%s\") count=%ld\n", section, output_char);
+}
+
+void ic_measureend()
+{
+    ic_check( 4 );
+    remove_ending_comma_if_any();
+    if (!subended) {
+         output_char += sprintf(&output[output_char], "   \n");
+    }
+    subended = 0;
+    DEBUG fprintf(stderr, "ic_measureend()\n");
+}
+
+/* Note this added a further tag to the measurement of the "resource_name" */
+/* measurement might be "disks" */
+/* sub might be "sda1", "sdb1", etc */
+void ic_sub(char *resource)
+{
+    int i;
+
+    ic_check( strlen(saved_section) + strlen(influx_tags) +strlen(saved_sub) + strlen(resource) + 9);
+
+    /* remove previously added section */
+    if (first_sub) {
+	for (i = output_char - 1; i > 0; i--) {
+	    if (output[i] == '\n') {
+		output[i + 1] = 0;
+		output_char = i + 1;
+		break;
+	    }
+	}
+    }
+    first_sub = 0;
+
+    /* remove the trailing s */
+    strcpy(saved_sub, saved_section);
+    if (saved_sub[strlen(saved_sub) - 1] == 's') {
+	saved_sub[strlen(saved_sub) - 1] = 0;
+    }
+    output_char += sprintf(&output[output_char], "%s,%s,%s_name=%s ", saved_section, influx_tags, saved_sub, resource);
+    subended = 0;
+    DEBUG fprintf(stderr, "ic_sub(\"%s\") count=%ld\n", resource, output_char);
+}
+
+void ic_subend()
+{
+    ic_check( 4 );
+    remove_ending_comma_if_any();
+    output_char += sprintf(&output[output_char], "   \n");
+    subended = 1;
+    DEBUG fprintf(stderr, "ic_subend()\n");
+}
+
+void ic_long(char *name, long long value)
+{
+    ic_check( strlen(name) + 16 + 4 );
+    output_char += sprintf(&output[output_char], "%s=%lldi,", name, value);
+    DEBUG fprintf(stderr, "ic_long(\"%s\",%lld) count=%ld\n", name, value, output_char);
+}
+
+void ic_double(char *name, double value)
+{
+    ic_check( strlen(name) + 16 + 4 );
+    if (isnan(value) || isinf(value)) { /* not-a-number or infinity */
+	DEBUG fprintf(stderr, "ic_double(%s,%.1f) - nan error\n", name, value);
+    } else {
+	output_char += sprintf(&output[output_char], "%s=%.3f,", name, value);
+	DEBUG fprintf(stderr, "ic_double(\"%s\",%.1f) count=%ld\n", name, value, output_char);
+    }
+}
+
+void ic_string(char *name, char *value)
+{
+    int i;
+    int len;
+
+    ic_check( strlen(name) + strlen(value) + 4 );
+    len = strlen(value);
+    for (i = 0; i < len; i++) 	/* replace problem characters and with a space */
+	if (value[i] == '\n' || iscntrl(value[i]))
+	    value[i] = ' ';
+    output_char += sprintf(&output[output_char], "%s=\"%s\",", name, value);
+    DEBUG fprintf(stderr, "ic_string(\"%s\",\"%s\") count=%ld\n", name, value, output_char);
+}
+
+void ic_push()
+{
+    char header[1024];
+    char result[1024];
+    char buffer[1024 * 8];
+    int ret;
+    int i;
+    int total;
+    int sent;
+    int code;
+
+    if (output_char == 0)	/* nothing to send so skip this operation */
+	return;
+    if (influx_port) {
+	DEBUG fprintf(stderr, "ic_push() size=%ld\n", output_char);
+	if (create_socket() == 1) {
+
+	    sprintf(buffer, "POST /write?db=%s&u=%s&p=%s HTTP/1.1\r\nHost: %s:%ld\r\nContent-Length: %ld\r\n\r\n",
+		    influx_database, influx_username, influx_password, influx_hostname, influx_port, output_char);
+	    DEBUG fprintf(stderr, "buffer size=%ld\nbuffer=<%s>\n", strlen(buffer), buffer);
+	    if ((ret = write(sockfd, buffer, strlen(buffer))) != strlen(buffer)) {
+			fprintf(stderr, "warning: \"write post to sockfd failed.\" errno=%d\n", errno);
+	    }
+	    total = output_char;
+	    sent = 0;
+	    if (debug == 2)
+		fprintf(stderr, "output size=%d output=\n<%s>\n", total, output);
+	    while (sent < total) {
+		ret = write(sockfd, &output[sent], total - sent);
+		DEBUG fprintf(stderr, "written=%d bytes sent=%d total=%d\n", ret, sent, total);
+		if (ret < 0) {
+		    fprintf(stderr, "warning: \"write body to sockfd failed.\" errno=%d\n", errno);
+		    break;
+		}
+		sent = sent + ret;
+	    }
+	    for (i = 0; i < 1024; i++) /* empty the buffer */
+		result[i] = 0;
+	    if ((ret = read(sockfd, result, sizeof(result))) > 0) {
+		result[ret] = 0;
+		DEBUG fprintf(stderr, "received bytes=%d data=<%s>\n", ret, result);
+		sscanf(result, "HTTP/1.1 %d", &code);
+		for (i = 13; i < 1024; i++)
+		    if (result[i] == '\r')
+			result[i] = 0;
+		if (debug == 2)
+		    fprintf(stderr, "http-code=%d text=%s [204=Success]\n", code, &result[13]);
+		if (code != 204)
+		    fprintf(stderr, "code %d -->%s<--\n", code, result);
+	    }
+	    close(sockfd);
+	    sockfd = 0;
+	    DEBUG fprintf(stderr, "ic_push complete\n");
+	} else {
+	    DEBUG fprintf(stderr, "socket create failed\n");
+	}
+    } else error("influx port is not set, bailing out");
+
+    output[0] = 0;
+    output_char = 0;
+}

ic.h

@@ -0,0 +1,22 @@
+/*
+ * Influx C (ic) client for data capture header file
+ * Developer: Nigel Griffiths.
+ * (C) Copyright 2021 Nigel Griffiths
+ */
+ void ic_influx_database(char *host, long port, char *db);
+ void ic_influx_userpw(char *user, char *pw);
+ void ic_tags(char *tags);
+
+ void ic_measure(char *section);
+ void ic_measureend();
+
+ void ic_sub(char *sub_name);
+ void ic_subend();
+
+ void ic_long(char *name, long long value);
+ void ic_double(char *name, double value);
+ void ic_string(char *name, char *value);
+
+ void ic_push();
+ void ic_debug(int level);
+

load_bpf.c

@@ -0,0 +1,75 @@
+#include <stdio.h>
+#include <bpf/bpf.h>
+#include <bpf/libbpf.h>
+#include <netinet/in.h>
+#include <sys/socket.h>
+#include <arpa/inet.h>
+#include "common.h"
+
+static void clean_obj(struct bpf_object *obj){
+    printf("Cleaning\n");
+    bpf_object__close(obj);
+}   
+int main(void){
+    const char *fileObj = "tp_tcp.o";
+    struct bpf_object *obj;
+    struct bpf_program *program;
+    struct bpf_map *map;
+    struct reset s_reset;
+    int err;
+    int map_fd;
+    long long stats;
+    int keys = 0;
+
+    obj = bpf_object__open_file(fileObj, NULL);
+    if (!obj){
+        printf("Failed to open %s\n", fileObj);
+        return -1;
+    }
+    //LIBBPF_OPTS(bpf_map_create_opts, opts, .map_flags = BPF_F_MMAPABLE);
+    map_fd = bpf_create_map(BPF_MAP_TYPE_HASH, sizeof(int), sizeof(struct reset), 4096, BPF_ANY);
+    printf("Create map: %d\n", map_fd);
+
+    err = bpf_object__load(obj);
+    printf("Object loaded: %d\n", err);
+    if (err){
+        printf("Failed to load object\n");
+        return -1;
+    }
+
+    program = bpf_object__find_program_by_name(obj, "tcp_retransmit");
+    if (!program){
+        printf("Failed to find the program\n");
+        return -1;
+    }
+
+    map = bpf_object__find_map_by_name(obj, "tcp_reset_stats");
+    if (!map){
+        printf("Failed to get the map\n");
+        clean_obj(obj);
+        return -1;
+    }
+
+    map_fd = bpf_object__find_map_fd_by_name(obj, "tcp_reset_stats");
+    printf("Map fd: %d\n", map_fd);
+
+    struct bpf_link *link = bpf_program__attach(program);
+    if (!link){
+        printf("Failed to attach the program\n");
+        return -1;
+    }
+
+    while(1){
+        int e = bpf_map_lookup_elem(map_fd, &keys, &s_reset);
+        if (e == 0){
+            //printf("%lld\n", stats);
+            struct in_addr *src = (struct in_addr*)&s_reset.saddr;
+            struct in_addr *dest = (struct in_addr*)&s_reset.daddr;
+            //struct in_addr src = s_reset.saddr;
+            printf("Sport: %d; dport: %d %s %s\n", s_reset.sport, s_reset.dport, inet_ntoa(*src), inet_ntoa(*dest));
+        }
+         
+    }
+
+    return 0;
+}

main.c

@@ -0,0 +1,12 @@
+#define BPF_NO_GLOBAL_DATA
+#include <stdio.h>
+#include <unistd.h>
+#include <linux/bpf.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+
+int main(int argc, char *argv[], char *argp[]){
+    printf("Hello world\n");
+    execve("/usr/bin/ls", argv, argp);
+    return 0;
+}

tp_tcp.c

@@ -0,0 +1,86 @@
+#define BPF_NO_GLOBAL_DATA
+//#define __TARGET_ARCH_x86
+#include "vmlinux.h"
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+#include <bpf/bpf_core_read.h>
+#include "common.h"
+
+char LICENSE[] SEC("license") = "Dual BSD/GPL";
+
+
+struct ctx_reset {
+    __u16 common_type;
+    __u8 common_flags;
+    __u8 common_count;
+    __s32 pid;
+
+    const void *skaddr;
+    __u16 sport; 
+    __u16 dport;
+    __u16 family;
+    __u8 saddr[4];
+    __u8 daddr[4];
+    __u8 saddr_v6[16];
+    __u8 daddr_v6[16];
+    __u64 sock_cookie;
+};
+
+struct {
+//    __uint(type, BPF_MAP_TYPE_ARRAY);
+    __uint(type, BPF_MAP_TYPE_HASH);
+    __uint(max_entries, 4096);
+    __type(key, int);
+    __type(value, sizeof(struct reset));
+} tcp_reset_stats SEC(".maps");
+
+// sudo tcpdump -i any 'tcp[13] & 4 != 0' -n -> filter TCP reset flags
+
+/*
+ * Identify all tracepoint available
+ *   - cat /sys/kernel/tracing/available_events
+ * Enable an event:
+ *   - echo 'tcp_receive_reset' >> /sys/kernel/tracing/set_event -> important to add the '>>'
+ * Docs: https://docs.kernel.org/trace/events.html
+ * https://events.linuxfoundation.org/wp-content/uploads/2022/10/elena-zannoni-tracing-tutorial-LF-2021.pdf
+ * https://docs.kernel.org/trace/tracepoints.html
+ * Why we need to detect RST:
+ * When we scan the port, the scanner send an SYN flag and if the port is block, we receive a RST flag:
+ * listening on any, link-type LINUX_SLL2 (Linux cooked v2), snapshot length 262144 bytes
+10:48:28.531295 lo    In  IP localhost.43961 > localhost.tproxy: Flags [S], seq 2197047013, win 1024, options [mss 1460], length 0
+10:48:28.531306 lo    In  IP localhost.tproxy > localhost.43961: Flags [R.], seq 0, ack 2197047014, win 0, length 0
+ * But we can also block all receive RST: iptables -I INPUT -p tcp --dport <port> -j REJECT --reject-with tcp-reset
+ */
+
+//SEC("tp/tcp_retransmit_synack")
+SEC("tracepoint/tcp/tcp_receive_reset")
+//int tcp_retransmit(struct sock *sk){
+int tcp_retransmit(struct ctx_reset *ctx){
+    long long *stats;
+    struct reset *s_reset;
+    int keys = 0;
+    __u16 sport = 0;
+    __u16 dport = 0;
+
+    s_reset = bpf_map_lookup_elem(&tcp_reset_stats, &keys);
+    if (!s_reset)
+        return 0;
+
+    //*stats += 1;
+    sport = ctx->sport;
+    dport = ctx->dport;
+    s_reset->saddr[0] = ctx->saddr[0];
+    s_reset->saddr[1] = ctx->saddr[1];
+    s_reset->saddr[2] = ctx->saddr[2];
+    s_reset->saddr[3] = ctx->saddr[3];
+    /*s_reset->daddr[0] = ctx->daddr[0];
+    s_reset->daddr[1] = ctx->daddr[1];
+    s_reset->daddr[2] = ctx->daddr[2];
+    s_reset->daddr[3] = ctx->daddr[3];*/
+    s_reset->sport = sport;
+    s_reset->dport = dport;
+
+    //bpf_printk("BPF detected TCP received reset %d - %d %d\n", *stats, dport, sport);
+    bpf_printk("BPF detected TCP received reset %d %d\n", dport, sport);
+    return 0;
+}