Delete reports

This commit is contained in:
geoffrey 2024-06-22 14:30:58 +02:00
parent 1e36568e44
commit 27271d2bb6
16 changed files with 47 additions and 684 deletions

4
config

@ -1,2 +1,2 @@
api_key: f4c451920a7e41ec344e16e6d36a1b7951bf23a8d224b796cb08301e65bf3114
dnsbl: dnsbl.txt
api_key_vt: f4c451920a7e41ec344e16e6d36a1b7951bf23a8d224b796cb08301e65bf3114
api_key_emailrep: foo

@ -1,252 +0,0 @@
#!/usr/bin/venv python
# -*- coding: utf-8 -*-
from argparse import ArgumentParser
from scapy.all import *
from scapy.layers.dns import DNSQR, DNSRR, DNS
import matplotlib.pyplot as plt
import numpy as np
import requests
import re
from reports import generateHtmlReport, createReportsDirectory, getTodayDate
from tunneling import tunnelingDNSAttacks
from config import VT_ATTRIBUTES_MAPPING
def _getType(t):
"""
This function identify the message type according to the RFC 1035:
https://datatracker.ietf.org/doc/html/rfc1035#section-3.2.2
"""
rtype = {
1: 'A', 2: 'NS', 3: 'MD', 4: 'MF',
5: 'CNAME', 6: 'SOA', 7: 'MB', 8: 'MG', 9: 'MR',
10: 'NULL', 11: 'WKS', 12: 'PTR', 13: 'HINFO', 14: 'MINFO',
15: 'MX', 16: 'TXT',
28: 'AAAA'
}
try:
return rtype[t]
except KeyError:
return t
def readPcapFile(pcap):
"""
This function read the pcap file with scapy
"""
data = None
try:
data = rdpcap(pcap)
except Scapy_Exception:
print(f"Failed to read {pcap}")
return None
return data
def _privateIP(ip):
"""
This function check if the ip is from RFC 1918
https://stackoverflow.com/questions/2814002/private-ip-address-identifier-in-regular-expression
"""
res = False
rfs1918 = [
'(^10\.)',
'(^172\.1[6-9]\.)',
'(^172\.2[0-9]\.)',
'(^172\.3[0-1]\.)',
'(^192\.168\.)',
]
for privateIP in rfs1918:
if re.search(privateIP, ip):
res = True
return res
def getIPVirusTotal(api_key, ip, report):
"""
This function get information of the IP
"""
url = f"https://www.virustotal.com/api/v3/ip_addresses/{ip}"
headers = {
'x-apikey': api_key,
}
res = requests.get(url, headers=headers).json()
data = dict()
data['ip'] = ip
if 'error' in res:
report.append({
'error': res['error']['message'],
'ip': ip
})
return
vt = res['data']['attributes']
for entry in VT_ATTRIBUTES_MAPPING.keys():
if entry in vt:
try:
data[entry] = vt[entry]
except KeyError:
data[entry] = 'Unknown'
report.append(data)
def main():
args = checkArguments()
report = {}
report['queries'] = list()
report['domains'] = list()
report['graphics'] = dict()
report['ip'] = list()
report['vt'] = list()
report['dnstunneling'] = dict()
if not args.file:
print("Please, specify the option --file")
exit(0)
if not args.config:
print("Please, specify the config file ith the parameter --config")
exit(0)
# Read the config file
config = readConfigFile(args.config)
if config is None:
print("Failed to read the config file")
exit(0)
# Get the API key for VirusTotal
checkVt = True
try:
api_key = config["api_key"]
except KeyError:
print("Can't find the key in the config file. Bypass the check to VirusTotal")
checkVt = False
#exit(0)
# Check if DNS BlackList is specified
try:
dnsbl = config["dnsbl"]
except KeyError:
print("Can't find the dnsbl in the config file")
exit(0)
data = readPcapFile(args.file)
if data is None:
print("Failed to read the pcap file")
exit(0)
# For number of request into the domain
numRequest = dict()
numIPDst = dict()
allIPs = list()
allDomains = list()
print("Parsing DNS capture")
for d in data:
if d.haslayer(DNS):
# For DNS query
if isinstance(d.qd, DNSQR):
query = d.qd.qname
qtype = _getType(d.qd.qtype)
src = d[IP].src
dst = d[IP].dst
# print(f"{d[DNS].id}; SRC: {src}; DST: {dst}, Query: {query}; TYPE: {qtype}; Size: {len(d[DNS])}")
report['queries'].append({
'id': d[DNS].id,
'src': src,
'dst': dst,
'query': query,
'type': qtype,
'len': len(d[DNS]),
})
# Count the number of request for a domain
if query not in numRequest:
numRequest[query] = 1
else:
numRequest[query] += 1
# Count the number of IP dst
if dst not in numIPDst:
numIPDst[dst] = 1
else:
numIPDst[dst] += 1
if not _privateIP(dst) and dst not in allIPs:
allIPs.append(dst)
allDomains.append({
'query': query,
'src': src,
'len': len(d[DNS]),
})
# For DNS response
if isinstance(d.an, DNSRR):
ans = d.an.rrname
atype = _getType(d.an.type)
src = d[IP].src
dst = d[IP].dst
# print(f"{d[DNS].id}; SRC: {src}; DST: {dst}, Answer: {ans}; TYPE: {atype}")
report['queries'].append({
'id': d[DNS].id,
'src': src,
'dst': dst,
'answer': ans,
'type': atype,
'len': len(d[DNS]),
})
# Create directory for the report
print("Generating directory for reports")
reportName = createReportsDirectory()
# Draw matplotlib
today = getTodayDate()
fname_domain = f'domain_{today}.png'
fname_ip = f'ip_{today}.png'
report['graphics']['domain'] = fname_domain
report['graphics']['ip'] = fname_ip
x = list(numRequest.keys())[0:20]
y = list(numRequest.values())[0:20]
plt.figure(figsize=(15, 5))
plt.barh(x, y)
#plt.show()
plt.savefig(
fname=f"reports/{today}/{fname_domain}",
dpi='figure',
format='png'
)
x = list(numIPDst.keys())[0:20]
y = list(numIPDst.values())[0:20]
plt.figure(figsize=(10, 5))
plt.barh(x, y, 0.5)
#plt.show()
plt.savefig(
fname=f"reports/{today}/{fname_ip}",
dpi='figure',
format='png'
)
# Identify DNS Tunneling attacks
print("Analyzing DNS capture for identifying DNS Tunneling")
tunnelingDNSAttacks(report['dnstunneling'], allDomains, dnsbl)
# For external IP, use curl to VirusTotal for analyzing the IP
checkVt = False
if checkVt:
print("Getting IP informations to VirusTotal")
for ip in allIPs:
getIPVirusTotal(api_key, ip, report['vt'])
# We generating the report
print("Generating the report")
generateHtmlReport(report)
print(f"Report generated at this directory: {reportName}")
if __name__ == "__main__":
main()

@ -11,6 +11,15 @@ class DNSInformations:
def __init__(self, api_key, fqdn):
self._fqdn = fqdn
def checkDomainExist(self):
try:
res_query = dns.resolver.resolve(self._fqdn, 'NS')
except dns.resolver.NoAnswer:
return False
except dns.resolver.NXDOMAIN:
return False
return True
def whois(self):
report = dict()
w = whois.whois(self._fqdn)

7
emailchecker.py Normal file

@ -0,0 +1,7 @@
#!/usr/bin/env python3
class EmailChecker:
def __init__(self, key):
self.headers = {
'Key': key,
}

37
main.py

@ -16,18 +16,18 @@ def checkArguments():
parser.add_argument('-c', '--config', help='Config file')
parser.add_argument('--dns', help='Get domain name information', action="store_true")
# For dns command
parser.add_argument('--dnsattacks', help='Parse DNS pcap file')
parser.add_argument('--domain', help='Get domain name information')
parser.add_argument('--host', help='Get domain name information')
parser.add_argument('--ip', help='Get IP information')
# For hash command
parser.add_argument('--hash', help='Hash file', action='store_true')
parser.add_argument('--hashfile', help='Hash file', action='store_true')
parser.add_argument('--scanvt', help='If specified, scan the hash with VirusTotal', action='store_true')
parser.add_argument('--md5', help='Hash file')
parser.add_argument('--sha1', help='Hash file')
parser.add_argument('--sha256', help='Hash file')
parser.add_argument('--sha384', help='Hash file')
parser.add_argument('--sha512', help='Hash file')
parser.add_argument('--hash', help='Get information about the hash')
return parser.parse_args()
@ -38,16 +38,17 @@ def usage():
print("A tool for SOC analyst\n")
print("Usage: main.py [COMMAND]")
print("-c PATH, --config PATH\t\tConfig file - mandatory")
print("--hash FILE\t\t\tHash the file and check in VirusTotal")
print("--hashfile\t\t\tHash the file and check in VirusTotal")
print("--hash HASH\t\tAnalyse the hash from VirusTotal")
print("--dns \t\t\t\tGet information regarding the domain with whois and VirusTotal")
print("--dnsattacks FILE\t\tParse the DNS pcap file and identify some DNS attacks")
print("--email\t\t\t\tGet informations about an email and check if has been compromised")
print("\n--dns command:")
print("\t --domain FQDN\t\tScan and get domain information")
print("\t --host HOST\t\tScan and get host information")
print("\t --ip IP\t\tScan and get IP information")
print("\n--hash command:")
print("\n--hashfile command:")
print("\t --md5 FILE\t\tGet the MD5 hash of the file")
print("\t --sha1 FILE\t\tGet the SHA256 of the file")
print("\t --sha256 FILE\t\tGet the SHA256 of the file")
@ -110,8 +111,8 @@ def main():
pass
if args.ip:
_parsingIP(config, args.ip, report)
# Analyse hash
if args.hash:
# Analyse hash file
if args.hashfile:
h = Hash()
dispatcher = {
"MD5": h.hashMd5,
@ -142,13 +143,16 @@ def main():
res = dispatcher[hashType](filename)
print(f"{hashType} hash: {res}")
res = "6548eec09f4d8bc6514bee3e5452541c"
if args.scanvt:
_parsingHash(config, res, report)
if args.scanvt:
_parsingHash(config, res, report)
# Analyse the hash
if args.hash:
_parsingHash(config, args.hash, report)
def _parsingHash(config, h, report):
report = dict()
vt = VT(config['api_key'])
vt = VT(config['api_key_vt'])
vt.getRateFromHash(h, report)
print("\n----------------------")
@ -181,7 +185,7 @@ def _parsingHash(config, h, report):
def _parsingIP(config, ip, report):
# Check if it's an IP address
vt = VT(config['api_key'])
vt = VT(config['api_key_vt'])
try:
ip_obj = ipaddress.ip_address(ip)
except ValueError:
@ -207,8 +211,13 @@ def _parsingIP(config, ip, report):
print(f"Cannot find the key {vt}")
def _parsingDomain(config, fqdn, report):
vt = VT(config['api_key'])
dns = DNS(config['api_key'], fqdn)
vt = VT(config['api_key_vt'])
dns = DNS(config['api_key_vt'], fqdn)
# Check if domain exist
if not dns.checkDomainExist():
print(f"The domain {fqdn} do not exist")
return
print("----------------------")
print("| resolving |")

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

@ -1,56 +0,0 @@
<!doctype html>
<html lang="en" data-bs-theme="auto">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="description" content="">
<meta name="author" content="Mark Otto, Jacob Thornton, and Bootstrap contributors">
<meta name="generator" content="Hugo 0.112.5">
<title>{{ data['title'] }}</title>
<link rel="canonical" href="https://getbootstrap.com/docs/5.3/examples/jumbotron/">
<link href="../templates/bootstrap.min.css" rel="stylesheet">
</head>
<body>
<main>
<div class="container py-4">
<header class="pb-3 mb-4 border-bottom">
<span class="d-flex align-items-center text-body-emphasis text-decoration-none; fs-4">Report</span>
</header>
{% include 'nav.html.j2' %}
<div class="p-5 mb-4 bg-body-tertiary rounded-3">
<div class="container-fluid py-5">
<h1 class="display-5 fw-bold">Report - DNS Tunneling</h1>
<p class="col-md-8 fs-4">Report of the day... blablabla</p>
</div>
</div>
<h2>DNS Blacklist</h2>
{% for item in data['dnstunneling']['blackDomain'] %}
<h3>{{ item }}</h3>
<p>Sources</p>
<ul>
{% for src in data['dnstunneling']['blackDomain'][item] %}
<li>{{ src }}</li>
{% endfor %}
</ul>
{% endfor %}
<h2>DNS datagram oversized</h2>
<p>In this section, we analyzed all sources who done DNS request with datagram oversized (200 bytes) and that can be a DNS tunneling.</p>
{% for item in data['dnstunneling']['oversized'] %}
<h3>{{ item }}</h3>
<p>Sources</p>
<ul>
{% for entry in data['dnstunneling']['oversized'][item] %}
<li>{{ entry['source'] }} ({{ entry['len'] }} bytes)</li>
{% endfor %}
</ul>
{% endfor %}
{% include 'footer.html.j2' %}

@ -1,11 +0,0 @@
<footer class="pt-3 mt-4 text-body-secondary border-top">
&copy; {{ data['year'] }}
</footer>
</div>
</main>
<script src="./templates/bootstrap.bundle.min.js"></script>
</body>
</html>

@ -1,36 +0,0 @@
<!doctype html>
<html lang="en" data-bs-theme="auto">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="description" content="">
<meta name="author" content="Mark Otto, Jacob Thornton, and Bootstrap contributors">
<meta name="generator" content="Hugo 0.112.5">
<title>{{ data['title'] }}</title>
<link rel="canonical" href="https://getbootstrap.com/docs/5.3/examples/jumbotron/">
<link href="../templates/bootstrap.min.css" rel="stylesheet">
</head>
<body>
<main>
<div class="container py-4">
<header class="pb-3 mb-4 border-bottom">
<span class="d-flex align-items-center text-body-emphasis text-decoration-none; fs-4">Report</span>
</header>
{% include 'nav.html.j2' %}
<div class="p-5 mb-4 bg-body-tertiary rounded-3">
<div class="container-fluid py-5">
<h1 class="display-5 fw-bold">Report - graphics</h1>
<p class="col-md-8 fs-4">Report of the day... blablabla</p>
</div>
</div>
<h3>Domains (20 first entries)</h3>
<img src="{{ data['graphics']['domain'] }}" title="Graphic" />
<h3>Destinations IP (20 first entries)</h3>
<img src="{{ data['graphics']['ip'] }}" title="Graphic" />
{% include 'footer.html.j2' %}

@ -1,113 +0,0 @@
<!doctype html>
<html lang="en" data-bs-theme="auto">
<head><script src="../assets/js/color-modes.js"></script>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="description" content="">
<meta name="author" content="Mark Otto, Jacob Thornton, and Bootstrap contributors">
<meta name="generator" content="Hugo 0.112.5">
<title>Jumbotron example · Bootstrap v5.3</title>
<link rel="canonical" href="https://getbootstrap.com/docs/5.3/examples/jumbotron/">
<link href="./templates/bootstrap.min.css" rel="stylesheet">
<style>
.bd-placeholder-img {
font-size: 1.125rem;
text-anchor: middle;
-webkit-user-select: none;
-moz-user-select: none;
user-select: none;
}
@media (min-width: 768px) {
.bd-placeholder-img-lg {
font-size: 3.5rem;
}
}
.b-example-divider {
width: 100%;
height: 3rem;
background-color: rgba(0, 0, 0, .1);
border: solid rgba(0, 0, 0, .15);
border-width: 1px 0;
box-shadow: inset 0 .5em 1.5em rgba(0, 0, 0, .1), inset 0 .125em .5em rgba(0, 0, 0, .15);
}
.b-example-vr {
flex-shrink: 0;
width: 1.5rem;
height: 100vh;
}
.bi {
vertical-align: -.125em;
fill: currentColor;
}
.nav-scroller {
position: relative;
z-index: 2;
height: 2.75rem;
overflow-y: hidden;
}
.nav-scroller .nav {
display: flex;
flex-wrap: nowrap;
padding-bottom: 1rem;
margin-top: -1px;
overflow-x: auto;
text-align: center;
white-space: nowrap;
-webkit-overflow-scrolling: touch;
}
.btn-bd-primary {
--bd-violet-bg: #712cf9;
--bd-violet-rgb: 112.520718, 44.062154, 249.437846;
--bs-btn-font-weight: 600;
--bs-btn-color: var(--bs-white);
--bs-btn-bg: var(--bd-violet-bg);
--bs-btn-border-color: var(--bd-violet-bg);
--bs-btn-hover-color: var(--bs-white);
--bs-btn-hover-bg: #6528e0;
--bs-btn-hover-border-color: #6528e0;
--bs-btn-focus-shadow-rgb: var(--bd-violet-rgb);
--bs-btn-active-color: var(--bs-btn-hover-color);
--bs-btn-active-bg: #5a23c8;
--bs-btn-active-border-color: #5a23c8;
}
.bd-mode-toggle {
z-index: 1500;
}
</style>
</head>
<body>
<main>
<div class="container py-4">
<header class="pb-3 mb-4 border-bottom">
<span class="d-flex align-items-center text-body-emphasis text-decoration-none; fs-4">Report</span>
</header>
<div class="p-5 mb-4 bg-body-tertiary rounded-3">
<div class="container-fluid py-5">
<h1 class="display-5 fw-bold">Report</h1>
<p class="col-md-8 fs-4">Report of the day... blablabla</p>
</div>
</div>
{% for plugin in data['plugins'] %}
{% include plugin %}
{% endfor %}
{% include 'footer.html.j2' %}

@ -1,16 +0,0 @@
<nav>
<ul class="nav nav-tabs">
<li class="nav-item">
<a class="nav-link active" aria-current="page" href="reports_queries.html">Queries</a>
</li>
<li class="nav-item">
<a class="nav-link" href="reports_graphics.html">Graphics</a>
</li>
<li class="nav-item">
<a class="nav-link" href="reports_vt.html">VirusTotal</a>
</li>
<li class="nav-item">
<a class="nav-link" href="reports_dns_tunneling.html">DNS Tunneling</a>
</li>
</ul>
</nav>

@ -1,43 +0,0 @@
<!doctype html>
<html lang="en" data-bs-theme="auto">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="description" content="">
<meta name="author" content="Mark Otto, Jacob Thornton, and Bootstrap contributors">
<meta name="generator" content="Hugo 0.112.5">
<title>{{ data['title'] }}</title>
<link rel="canonical" href="https://getbootstrap.com/docs/5.3/examples/jumbotron/">
<link href="../templates/bootstrap.min.css" rel="stylesheet">
</head>
<body>
<main>
<div class="container py-4">
<header class="pb-3 mb-4 border-bottom">
<span class="d-flex align-items-center text-body-emphasis text-decoration-none; fs-4">Report</span>
</header>
{% include 'nav.html.j2' %}
<div class="p-5 mb-4 bg-body-tertiary rounded-3">
<div class="container-fluid py-5">
<h1 class="display-5 fw-bold">Report - queries</h1>
<p class="col-md-8 fs-4">Report of the day... blablabla</p>
</div>
</div>
{% for item in data['queries'] %}
Packet id: {{ item['id'] }}<br />
<span style="margin-left:15pt;">Src: {{ item['src'] }};
Dest: {{ item['dst'] }};
{% if 'query' in item %}
Query: {{ item['query'] }};
{% elif 'answer' in item %}
Answer: {{ item['answer'] }};
{% endif %}
Type: {{ item['type'] }}
Len: {{ item['len'] }}</span><br /><br />
{% endfor %}
{% include 'footer.html.j2' %}

@ -1,44 +0,0 @@
<!doctype html>
<html lang="en" data-bs-theme="auto">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="description" content="">
<meta name="author" content="Mark Otto, Jacob Thornton, and Bootstrap contributors">
<meta name="generator" content="Hugo 0.112.5">
<title>{{ data['title'] }}</title>
<link rel="canonical" href="https://getbootstrap.com/docs/5.3/examples/jumbotron/">
<link href="../templates/bootstrap.min.css" rel="stylesheet">
</head>
<body>
<main>
<div class="container py-4">
<header class="pb-3 mb-4 border-bottom">
<span class="d-flex align-items-center text-body-emphasis text-decoration-none; fs-4">Report</span>
</header>
{% include 'nav.html.j2' %}
<div class="p-5 mb-4 bg-body-tertiary rounded-3">
<div class="container-fluid py-5">
<h1 class="display-5 fw-bold">Report</h1>
<p class="col-md-8 fs-4">Get IP information for each public IP addresses from the DNS capture. Get informations from <a href='https://www.virustotal.com' title='VirusTotal'/>VirusTotal</a></p>
</div>
</div>
{% for entry in data['vt'] %}
<h3>{{ entry['ip']}}</h3>
{% if 'error' in entry %}
<p style='color:red'>{{ entry['error'] }}</p>
{% else %}
<ul>
{% for vt in entry['data'] %}
<li><strong>{{ vt }}</strong>: {{ entry['data'][vt] }}</li>
{% endfor %}
</ul>
{% endif %}
{% endfor %}
{% include 'footer.html.j2' %}

@ -1,82 +0,0 @@
#!/usr/bin/venv python
# -*- coding: utf-8 -*-
def tunnelingDNSAttacks(report, queries, dnsbl):
"""
This function identify if we have a DNS tunneling
We can identify the payload size (512 bytes) and for a specific domain, how many occurs it appear.
For the payload size, according to the RFC 1035: https://datatracker.ietf.org/doc/html/rfc1035#section-2.3.4
The maximum UDP message is 512 or less.
If we detect a lot of request to a specific domain, that can be a DNS tunneling
"""
report['blackDomain'] = _blackDomain(queries, dnsbl)
report['oversized'] = _oversized(queries)
def _blackDomain(queries, dnsbl) -> list:
""""
This function identify if a dns domain is in a blacklist
Return the report
"""
report = dict()
# Read the black domain list
dataDnsbl = list()
with open(dnsbl, 'r') as f:
for line in f:
dataDnsbl.append(line.replace("\n", ""))
for query in queries:
domain = _splitDomain(query['query'])
# Check if in dns blacklist
if domain in dataDnsbl:
if domain not in report:
report[domain] = list()
report[domain].append(query['src'])
else:
if query['src'] not in report[domain]:
report[domain].append(query['src'])
return report
def _oversized(queries) -> dict:
"""
This function analyzed the payload size and if it's oversize, that can be a DNS tunneling
"""
report = dict()
for query in queries:
if query['len'] > 220:
domain = _splitDomain(query['query'])
if domain not in report:
report[domain] = list()
report[domain].append({
'source': query['src'],
'len': query['len'],
})
else:
if query['src'] not in report[domain]:
report[domain].append({
'source': query['src'],
'len': query['len'],
})
return report
def _splitDomain(query) -> str:
"""
This function split the query for identifying the domain
"""
# Split to get the country code and the TLD
domainSplitted = str(query).split(".")
l = len(domainSplitted)
tld = domainSplitted[l - 3:l - 2]
cc = domainSplitted[l - 2 :l - 1]
try:
domain = f"{tld[0]}.{cc[0]}"
except IndexError:
print(f"Failed to parse the domain {query}")
domain = None
return domain

8
vt.py

@ -85,8 +85,12 @@ class VT:
report['results']['file']['filetype'] = attributes['detectiteasy']['filetype']
report['results']['file']['size'] = attributes['size']
report['results']['file']['extension'] = attributes['type_extension']
report['results']['file']['first_seen'] = \
datetime.fromtimestamp(attributes['first_seen_itw_date'])
try:
report['results']['file']['first_seen'] = \
datetime.fromtimestamp(attributes['first_seen_itw_date'])
except KeyError:
report['results']['file']['first_seen'] = "Unknown"
report['results']['file']['last_analysis'] = \
datetime.fromtimestamp(attributes['last_analysis_date'])