File: //opt/imunify360/venv/lib/python3.11/site-packages/im360/model/incident.py
import ipaddress
import time
from typing import Dict, List, Optional, Set
from peewee import (
JOIN,
Case,
CharField,
CompositeKey,
FloatField,
ForeignKeyField,
IntegerField,
IntegrityError,
PrimaryKeyField,
TextField,
prefetch,
)
from playhouse.shortcuts import model_to_dict
from defence360agent.model import Model, instance
from defence360agent.model.simplification import apply_order_by
from im360.contracts.config import (
ControlPanelProtector,
CpHulkSensor,
ModsecSensor,
OssecSensor,
)
from im360.model.country import Country
from im360.model.firewall import IPList, IPListPurpose
ossec_to_modsec_severity = {
1: 7, # debug level
2: 6,
3: 5,
4: 4, # default for UI filtering
5: 4,
6: 3,
7: 3,
8: 3,
9: 3,
10: 3,
11: 3,
12: 2,
13: 2,
14: 1,
15: 0, # emergency level
}
class _SafeCharField(CharField):
def adapt(self, value):
return super().adapt(value.encode("utf-8", errors="ignore"))
class Incident(Model):
"""Security-related events that happened on the server."""
# supplying each field with null=True to be consistent
# with previously used create table sql:
# CREATE TABLE incident (
# id INTEGER PRIMARY KEY,
# plugin TEXT,
# rule TEXT,
# timestamp REAL,
# retries INTEGER,
# severity NUMERIC,
# name TEXT,
# description TEXT,
# abuser TEXT
# );
id = IntegerField(primary_key=True, null=True)
#: The name of the sensor used to detect an incident, e.g. modsec, cl_dos.
plugin = CharField(null=True)
#: The ID of the rule.
rule = CharField(null=True)
#: Timestamp when the incident happened, or at least was detected.
timestamp = FloatField(null=True)
#: How many times it happened - incidents are aggregated over
#: a short period preserving most of the fields, except for the exact
#: :attr:`timestamp` and :attr:`description`.
retries = IntegerField(null=True)
#: How significant the threat is.
#: All plugins/sensors are brought to the scale roughly matching the `OSSEC
#: classification <https://www.ossec.net/docs/manual/rules-decoders/
#: rule-levels.html#rules-classification>`_
severity = IntegerField(null=True)
#: A human-readable name of the triggered rule.
name = CharField(null=True)
#: A detailed description of the event.
description = _SafeCharField(null=True)
#: The IP that has caused the incident, if applicable.
abuser = CharField(null=True)
#: A reference to country code and name for the IP, based on GeoDB data.
country = CharField(null=True, column_name="country_id")
#: A domain name related to the incident, if available.
domain = TextField(null=True, default=None)
class Meta:
database = instance.db
db_table = "incident"
indexes = (
(("timestamp",), False),
(("country",), False),
)
schema = "resident"
class OrderBy:
@staticmethod
def severity():
max_ossec_severity = max(ossec_to_modsec_severity.keys())
ossec_cases = tuple(
(
ossec,
modsec
+ (max_ossec_severity + 1 - ossec)
/ (max_ossec_severity + 1),
)
# sort ossec's incidents correctly when
# modsec's severity equivalents equal
for ossec, modsec in ossec_to_modsec_severity.items()
)
return (
Case(
Incident.plugin,
(
(
OssecSensor.PLUGIN_ID,
Case(Incident.severity, ossec_cases, 0),
),
(
CpHulkSensor.PLUGIN_ID,
Case(Incident.severity, ossec_cases, 0),
),
(ModsecSensor.PLUGIN_ID, Incident.severity),
),
100,
),
) # incidents without severity to the end
@classmethod
def _accept_severity(cls, severity):
return (
(
(
(cls.plugin == OssecSensor.PLUGIN_ID)
| (cls.plugin == ControlPanelProtector.PLUGIN_ID)
| (cls.plugin == CpHulkSensor.PLUGIN_ID)
)
& (cls.severity >= severity)
)
| (
(cls.plugin == ModsecSensor.PLUGIN_ID)
& (cls.severity <= ossec_to_modsec_severity[severity])
)
| cls.severity.is_null()
)
@classmethod
def get_sorted_incident_list(
cls,
since=None,
to=None,
by_abuser_ip=None,
by_list=None,
limit=None,
offset=None,
severity=None,
by_country_code=None,
by_domains=None,
search=None,
order_by=None,
by_plugin=None,
):
"""
:param by_country_code: country code in form 'US => United States'
:param integer since: unixtime when records is began
:param integer to: unixtime when records is ended
:param str by_abuser_ip: full or part of IP, used for filtering
results by abuser's IP
:param str by_list: List of names of the appropriate ip list. Could be
'gray', 'white', 'black'.
:param int limit: limits the output with specified number of
incidents. The number greater than zero
:param int offset: offset for pagination
:param int severity: min log level (severity) to return.
:param str search: filter results by ip, name, description
:param list order_by: sorting orders
:param list of str by_domains: filter by panel user domains
:param str by_plugin: filter by plugin name, e.g. 'modsec', 'ossec'.
"""
if to is None:
to = time.time()
query = (
Incident.select(Incident, Country)
.join(
Country, JOIN.LEFT_OUTER, on=(Incident.country == Country.id)
)
.where(
(Incident.timestamp >= since)
& cls._accept_severity(severity)
& (Incident.timestamp <= to)
)
.order_by(Incident.timestamp.desc())
)
if by_domains is not None:
query = query.where(Incident.domain << by_domains)
if search is not None:
query = query.where(
Incident.name.contains(search)
| Incident.description.contains(search)
| Incident.domain.contains(search)
| Incident.abuser.contains(search)
)
if by_abuser_ip is not None:
query = query.where(Incident.abuser.contains(by_abuser_ip))
if by_country_code is not None:
query = query.where(Country.code == by_country_code)
if by_plugin is not None:
query = query.where(Incident.plugin == by_plugin)
# DEF-32239: listname/purpose come from a CIDR-aware lookup, not
# the buggy ``Incident.abuser == IPList.ip`` string-equality join.
# When ``by_list`` is set we must resolve abuser->listname BEFORE
# pagination so we can SQL-filter the incidents; otherwise we can
# defer until the page is materialised so we resolve only its
# abusers.
listname_filter: Optional[Set[str]] = (
{ln.upper() for ln in by_list} if by_list is not None else None
)
abuser_listname: Optional[Dict[str, Optional[str]]] = None
if listname_filter is not None:
candidate_abusers = {
row.abuser
for row in query.select(Incident.abuser).distinct()
if row.abuser
}
abuser_listname = cls._resolve_abuser_listnames(
candidate_abusers, listname_filter
)
matched_abusers = {
a for a, ln in abuser_listname.items() if ln is not None
}
if not matched_abusers:
return []
query = query.where(Incident.abuser << matched_abusers)
if order_by is not None:
query = apply_order_by(order_by, cls, query)
if offset is not None:
query = query.offset(offset)
if limit is not None:
query = query.limit(limit)
rows = list(query)
if abuser_listname is None:
abuser_listname = cls._resolve_abuser_listnames(
{r.abuser for r in rows if r.abuser}, None
)
return list(cls.mk_incident_iterator(rows, abuser_listname))
@classmethod
def mk_incident_iterator(
cls,
rows,
abuser_listname: Dict[str, Optional[str]],
):
for row in rows:
ln_upper = abuser_listname.get(row.abuser) if row.abuser else None
listname = ln_upper.lower() if ln_upper else None
purpose = (
IPListPurpose.listname2purpose(ln_upper).value
if ln_upper
else None
)
yield {
"id": row.id,
"plugin": row.plugin,
"rule": row.rule,
"timestamp": row.timestamp,
"times": row.retries,
"severity": row.severity,
"name": row.name,
"description": row.description,
"abuser": row.abuser,
"listname": listname,
"purpose": purpose,
"country": model_to_dict(Country.get(id=row.country))
if row.country
else {},
"domain": row.domain,
}
@classmethod
def _resolve_abuser_listnames(
cls,
abusers: Set[str],
listname_filter: Optional[Set[str]] = None,
) -> Dict[str, Optional[str]]:
"""Return ``{abuser_ip: highest-priority IPList listname covering it}``.
``None`` value if the abuser is not in any (allowed) list. One SQL
fetch of the candidate IPList rows + Python containment via
:py:meth:`ipaddress.IPv4Network.subnet_of` — avoids issuing one DB
query per abuser.
"""
if not abusers:
return {}
q = IPList.select(
IPList.network_address,
IPList.netmask,
IPList.version,
IPList.listname,
).where(~IPList.is_expired())
if listname_filter is not None:
q = q.where(IPList.listname.in_(list(listname_filter)))
# Materialise the network objects (and listname priorities) once,
# *not* once per abuser — ``IPList.ip_network`` is a non-cached
# ``@property`` that calls ``unpack_ip_network`` on every access.
priority = {ln: p for p, ln in IPList.IP_LIST_PRIORITIES}
iplist_entries = [
(row.ip_network, row.listname, priority.get(row.listname, -1))
for row in q
]
result: Dict[str, Optional[str]] = {}
for abuser in abusers:
try:
abuser_net = ipaddress.ip_network(abuser)
except ValueError:
result[abuser] = None
continue
best, best_prio = None, -1
abuser_version = abuser_net.version
for row_net, row_listname, row_prio in iplist_entries:
if row_net.version != abuser_version:
continue
if row_prio <= best_prio:
continue
if abuser_net.subnet_of(row_net):
best, best_prio = row_listname, row_prio
result[abuser] = best
return result
@staticmethod
def save_incident_list(data):
# number of rows to insert in one query
num_rows = 50
with instance.db.atomic():
for idx in range(0, len(data), num_rows):
Incident.insert_many(data[idx : idx + num_rows]).execute()
@classmethod
def _add_common_filters(cls, query, kwargs):
if "domain" in kwargs:
query = query.where(cls.domain == kwargs["domain"])
if "ip" in kwargs:
query = query.where(cls.abuser == kwargs["ip"])
if "attack_type" in kwargs:
query = query.where(cls.name == kwargs["attack_type"])
if "description" in kwargs:
query = query.where(
cls.description.contains(kwargs["description"])
)
return query
class DisabledRule(Model):
"""Provides a way to ignore certain rules."""
class Meta:
database = instance.db
db_table = "disabled_rules"
indexes = ((("plugin", "rule_id"), True),)
id = PrimaryKeyField()
#: The name of the sensor used to detect an incident, e.g. modsec, cl_dos.
plugin = CharField(null=False)
#: The ID of the rule.
rule_id = CharField(null=False)
#: A human-readable name of the rule.
#: Only used for UX, doesn't affect detection logic.
name = TextField(null=False)
@classmethod
def as_list(cls) -> List[Dict]:
return [
{
cls.plugin.name: rule.plugin,
cls.rule_id.name: rule.rule_id,
cls.name.name: rule.name,
}
for rule in cls.select()
]
@classmethod
def is_rule_ignored(cls, plugin, rule_id, domain=None):
try:
dr = cls.get(plugin=plugin, rule_id=rule_id)
if dr.domains:
return domain in (d.domain for d in dr.domains)
else:
return True
except cls.DoesNotExist:
pass
return False
@classmethod
def get_global_disabled(cls, plugin):
query = (
cls.select(cls.rule_id)
.join(DisabledRuleDomain, JOIN.LEFT_OUTER)
.where(
(cls.plugin == plugin) & (DisabledRuleDomain.domain >> None)
)
.dicts()
)
return [row["rule_id"] for row in query]
@classmethod
def get_domain_disabled(cls, plugin, domain):
query = (
cls.select(cls.rule_id)
.join(DisabledRuleDomain)
.where(cls.plugin == plugin, DisabledRuleDomain.domain == domain)
.dicts()
)
return [row["rule_id"] for row in query]
@classmethod
def fetch(cls, limit, offset=0, order_by=None):
rules_query = (
cls.select()
.order_by(cls.plugin, cls.rule_id)
.limit(limit)
.offset(offset)
)
if order_by is not None:
rules_query = apply_order_by(order_by, cls, rules_query)
domains_query = DisabledRuleDomain.select()
rules_with_domains_query = prefetch(rules_query, domains_query)
result = []
max_count = rules_query.count(clear_limit=True)
for rule in rules_with_domains_query:
item = {
"plugin": rule.plugin,
"id": rule.rule_id,
"name": rule.name,
"domains": None,
}
if rule.domains:
item["domains"] = [d.domain for d in rule.domains]
result.append(item)
return max_count, result
@classmethod
def store(self, plugin, id, name, domains):
try:
inserted_id = DisabledRule.insert(
plugin=plugin, rule_id=id, name=name
).execute()
except IntegrityError:
dr = DisabledRule.get(plugin=plugin, rule_id=id)
if domains:
for d in domains:
DisabledRuleDomain.create_or_get(
disabled_rule_id_id=dr.id, domain=d
)
else:
DisabledRuleDomain.delete().where(
DisabledRuleDomain.disabled_rule_id_id == dr.id
).execute()
else:
for d in domains:
DisabledRuleDomain.create(
disabled_rule_id_id=inserted_id, domain=d
)
class DisabledRuleDomain(Model):
"""Allows to disable rules for specific domains.
If there are no records in this table related to :class:`DisabledRule`,
then the rule is ignored for all domains.
Otherwise, the rule is ignored only for domains listed.
"""
disabled_rule_id_id = ForeignKeyField(
DisabledRule, backref="domains", on_delete="CASCADE"
)
#: The domain name, for which the rule must be disabled.
domain = CharField(null=False)
class Meta:
database = instance.db
db_table = "disabled_rules_domains"
primary_key = CompositeKey("disabled_rule_id_id", "domain")