File: //proc/self/root/opt/imunify360/venv/share/imunify360/scripts/rules_checker.py
"""
Usage /opt/imunify360/venv/bin/python3 /opt/imunify360/venv/share/imunify360/scripts/rules_checker.py <action>
choose an action from ACTIONS map
e.g.
/opt/imunify360/venv/bin/python3 rules_checker.py recreate
Actions:
- `recreate` - recreates rules if needed and checks ipsets consistent
- `clear` - waits RulesChecker stop and destroys all rules and ipsets
Actions based on lazy_init plugin
placed in im360.utils.lazy_init
"""
import itertools
import asyncio
import json
from pathlib import Path
import os
import sys
import time
import logging
import argparse
from defence360agent.internals import logger as lg
from defence360agent.internals.global_scope import g
from defence360agent.model import instance, tls_check, simplification, infected_domain
from defence360agent.contracts.config import Model, Merger
from im360.contracts.config import IPSET_LISTS_PATH
from im360.files import WHITELISTS, Index
from im360.internals.core import ip_versions
from im360.internals.strategy import Strategy
from im360.model import (
firewall,
incident,
messages_to_send,
proactive,
)
from im360.utils.lazy_init import (
RulesChecker,
RealProtector,
RULES_CHECK_IN_PROGRESS,
)
logger = logging.getLogger("rules-checker")
# go-resident reads from stdout in case we have errors
handler = logging.StreamHandler(sys.stdout)
handler.setLevel(logging.INFO)
logger.addHandler(handler)
STATE = {"last_ipset_check": 0.0}
DAY = 24 * 60 * 60
REAL_PROTECTOR_STATE = Path("/var/imunify360/.realprotector.state")
RULES_CHECKER_STATE = Path("/var/imunify360/.ruleschecker.state")
RULES_CHECKER_EXTERN_STATE = Path(
"/var/imunify360/.ruleschecker.extern_state.json"
)
REAL_PROTECTOR_STATE_JSON = Path("/var/imunify360/.realprotector.state.json")
class RealProtectorState:
"""RealProtector state to save and restore."""
def __init__(self, _ws, _pb_dmv, _pbm, _de, _lic):
self._webshield_status = _ws
self._port_blocking_deny_mode_values = _pb_dmv
self._port_blocking_mode = _pbm
self._dos_enabled = _de
self.last_ipset_check = _lic
def __str__(self) -> str:
return (
"RealProtectorState("
f"_webshield_status={self._webshield_status}, "
f"_port_blocking_mode={self._port_blocking_mode}, "
"_port_blocking_deny_mode_values"
f"={self._port_blocking_deny_mode_values},"
f"last_ipset_check={self.last_ipset_check},"
f"_dos_enabled={self._dos_enabled})"
)
def to_json(self) -> dict:
return {
"webshield_status": self._webshield_status,
"port_blocking_deny_mode_values": self._port_blocking_deny_mode_values,
"port_blocking_mode": self._port_blocking_mode,
"dos_enabled": self._dos_enabled,
"last_ipset_check": self.last_ipset_check,
}
@classmethod
def from_json(cls, data: dict) -> "RealProtectorState":
# webshield_status and port_blocking_deny_mode_values are tuples in
# RealProtector; the previous pickle round-trip preserved that type
# but JSON arrays decode as lists. RulesChecker compares them with
# != against the current tuple value, so without coercion every
# restart would see a spurious "change" and recreate firewall rules.
ws = data.get("webshield_status")
if isinstance(ws, list):
ws = tuple(ws)
pbdmv = data.get("port_blocking_deny_mode_values")
if isinstance(pbdmv, list):
pbdmv = tuple(pbdmv)
return cls(
_ws=ws,
_pb_dmv=pbdmv,
_pbm=data.get("port_blocking_mode"),
_de=data.get("dos_enabled", True),
_lic=data.get("last_ipset_check", 0),
)
class RulesCheckerState:
def __init__(
self, interface_conf, ipset_outdated_events, outdated_ipsets, versions
):
self._interface_conf = interface_conf
self._ipsets_outdated_events = ipset_outdated_events
self.outdated_ipsets = outdated_ipsets
self.versions = versions
def __str__(self) -> str:
return (
"RulesCheckerState("
f"_interface_conf={self._interface_conf}, "
f"_ipsets_outdated_events={self._ipsets_outdated_events})"
f"outdated_ipsets={self.outdated_ipsets})"
f"versions={self.versions})"
)
def make_external_state(self) -> dict:
versions_data = {}
for version, version_state in self.versions.items():
versions_data[version] = {
"transient_error_on_create": version_state.transient_error_on_create,
"errors": version_state.errors,
"next_try_time": version_state.next_try_time,
"running": version_state.running,
}
_outdated_ipsets = {
ver: list(sets) for ver, sets in self.outdated_ipsets.items()
}
return {
"versions": versions_data,
"outdated_ipsets": _outdated_ipsets,
# Persist interface config so recreate_rules_if_needed can
# detect interface changes across restarts; the pickle path
# used to do this implicitly via the pickled object graph.
"interface_conf": self._interface_conf,
}
async def _check_for_config_change(rc: RulesChecker, rp: RealProtector):
"""Checking that config state is consistent with the current state."""
rp._rules_checker = rc
await rp._on_config_update_unlocked(None)
async def recreate_rules(rc: RulesChecker, rp: RealProtector, **kwargs):
"""Recreates rules if needed and checks ipsets consistent."""
logger.info("Checking that need to recreate rules")
# TODO: check if we need to check it too often
# for Python implementation we do it only once per day
if time.time() - STATE["last_ipset_check"] < DAY:
logger.info("Skip ipsets check")
else:
await rc._check_ipsets_consistent()
STATE["last_ipset_check"] = time.time()
await rc.recreate_rules_if_needed()
logger.info("IP sets verification and initialization completed")
async def check_config_update(rc: RulesChecker, rp: RealProtector, **kwargs):
"""Checking config update."""
logger.info("Checking config update")
await _check_for_config_change(rc, rp)
logger.info("Completed")
async def check_global_whitelist_update(
rc: RulesChecker, rp: RealProtector, **kwargs
):
"""Checking config update."""
rp._rules_checker = rc
logger.info("Checking global whitelist update")
await rp.process_global_whitelist_update()
logger.info("Completed")
async def check_country_update(rc: RulesChecker, rp: RealProtector, **kwargs):
"""Checking config update."""
rp._rules_checker = rc
logger.info("Checking country list update")
await rp.process_country_list_update()
logger.info("Completed")
async def recreate_rules_on_strategy_change(
rc: RulesChecker, rp: RealProtector, **kwargs
):
"""Recreates rules if needed and checks ipsets consistent."""
logger.info("Checking that need to recreate rules on strategy change")
await rc.recreate_rules_if_needed()
logger.info(
"Firewall rules recreated due to StrategyChange %s", Strategy.current
)
async def check_ipsets_consistent(
rc: RulesChecker, rp: RealProtector, check_all=False, **kwargs
):
"""Check ipsets consistent."""
logger.info("Checking ipsets consistent")
await rc._check_ipsets_consistent(check_all)
STATE["last_ipset_check"] = time.time()
if any(sets for sets in rc.outdated_ipsets.values()):
await rc.recreate_rules_if_needed()
logger.info("Completed")
async def _stop_and_wait(rc: RulesChecker):
rc.should_stop()
await rc.wait()
async def clear_everything(rc: RulesChecker, rp: RealProtector, **kwargs):
"""Clear rules and ipsets on stop."""
logger.info("Clear rules and ipsets")
await _stop_and_wait(rc)
await rc.clear_everything()
logger.info("Completed")
async def clear_rules(rc: RulesChecker, rp: RealProtector, **kwargs):
"""Clear rules on stop."""
logger.info("Clear rules")
await _stop_and_wait(rc)
await rc.clear_rules()
logger.info("Completed")
async def force_recreate_rules_and_refill_ports_if_needed(
rc: RulesChecker, rp: RealProtector, refill=False, **kwargs
):
rp._rules_checker = rc
await rc.recreate_rules_if_needed(recreate_any_way=True)
logger.info("Firewall rules recreated due to ConfigUpdate")
if refill and await rp._refill_port_blocking_ipsets():
logger.info("Blocked ports ipsets reffiled")
async def refill_ports(rc: RulesChecker, rp: RealProtector, **kwargs):
rp._rules_checker = rc
if await rp._refill_port_blocking_ipsets():
logger.info("Blocked ports deny mode updated on ConfigUpdate")
def setup_environment():
"""Setup environment for rules checker."""
lg.reconfigure()
ip_versions.init()
instance.db.init(f"file:{Model.PATH}?mode=ro", uri=True)
instance.db.execute_sql("ATTACH ? AS resident", (Model.RESIDENT_PATH,))
instance.db.execute_sql("ATTACH ? AS ipsetlists", (IPSET_LISTS_PATH,))
models = list(itertools.chain(
*[
simplification.get_models(module)
for module in (
simplification,
firewall,
incident,
messages_to_send,
proactive,
infected_domain,
)
]
))
instance.db.bind(models)
if os.environ.get("DEBUG") == "true":
g.DEBUG = True
Index.add_type(WHITELISTS, "whitelist/v2", 0o770, 0o660, all_zip=True)
def restore_state(rp: RealProtector, rc: RulesChecker):
"""Restore RealProtector state."""
Strategy.current = Strategy.get()
# Remove legacy pickle files to prevent deserialization of arbitrary objects
for legacy in (REAL_PROTECTOR_STATE, RULES_CHECKER_STATE):
try:
if legacy.exists():
legacy.unlink()
except Exception:
pass
try:
if REAL_PROTECTOR_STATE_JSON.exists():
with REAL_PROTECTOR_STATE_JSON.open("r") as f:
rp_state = RealProtectorState.from_json(json.load(f))
rp._webshield_status = rp_state._webshield_status
rp._port_blocking_deny_mode_values = (
rp_state._port_blocking_deny_mode_values
)
rp._port_blocking_mode = rp_state._port_blocking_mode
# next is new functionality and _dos_enabled will be missing
# on startup. Since this is used to prevent adding connection
# tracking to netfilter we basically had this before set to
# True, that is why default value needs to be True
rp._dos_enabled = rp_state._dos_enabled
STATE["last_ipset_check"] = rp_state.last_ipset_check
except Exception as e:
logger.error("Failed to restore RealProtector state: %s", e)
try:
if RULES_CHECKER_EXTERN_STATE.exists():
with RULES_CHECKER_EXTERN_STATE.open("r") as f:
extern_state = json.load(f)
# Restore versions state from JSON
if "versions" in extern_state:
for version, vdata in extern_state["versions"].items():
if version in rc.versions:
vs = rc.versions[version]
vs.transient_error_on_create = vdata.get(
"transient_error_on_create", False
)
vs.errors = vdata.get("errors", 0)
vs.next_try_time = vdata.get("next_try_time", 0)
vs.running = True
# Restore outdated_ipsets from JSON
if "outdated_ipsets" in extern_state:
rc.outdated_ipsets = {
ver: set(sets)
for ver, sets in extern_state["outdated_ipsets"].items()
}
# Restore the active interface config so
# recreate_rules_if_needed can detect interface changes
# across restarts (the pickle path persisted this
# implicitly).
if "interface_conf" in extern_state:
rc.active_interface_conf = extern_state["interface_conf"]
except Exception as e:
logger.error("Failed to restore RulesChecker state: %s", e)
return rp, rc
def save_state(rp: RealProtector, rc: RulesChecker):
"""Save RealProtector state."""
rp_state = RealProtectorState(
rp._webshield_status,
rp._port_blocking_deny_mode_values,
rp._port_blocking_mode,
rp._dos_enabled,
STATE["last_ipset_check"],
)
rc_state = RulesCheckerState(
rc.active_interface_conf,
rc._ipsets_outdated_events,
rc.outdated_ipsets,
rc.versions,
)
with REAL_PROTECTOR_STATE_JSON.open("w") as f:
json.dump(rp_state.to_json(), f, indent=4)
with RULES_CHECKER_EXTERN_STATE.open("w") as f:
json.dump(rc_state.make_external_state(), f, indent=4)
ACTIONS = {
"recreate": recreate_rules,
"clear": clear_everything,
"clear-rules": clear_rules,
"config-update": check_config_update,
"global-whitelist-update": check_global_whitelist_update,
"country-update": check_country_update,
"strategy-change": recreate_rules_on_strategy_change,
"ipsets-consistent": check_ipsets_consistent,
"force-recreate-rules": force_recreate_rules_and_refill_ports_if_needed,
"refill-ports": refill_ports,
}
def parse_arguments():
"""Parse command line arguments."""
parser = argparse.ArgumentParser(
description="Imunify360 Rules Checker CLI",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog=__doc__,
)
parser.add_argument(
"action", choices=ACTIONS.keys(), help="Action to perform"
)
parser.add_argument(
"--refill-ports",
action="store_true",
help="refills ports in force-recreate-rules",
)
parser.add_argument(
"--check-all",
action="store_true",
help=(
"Check consistency off all ipsets on python side. Should be used"
" only in tests"
),
)
return parser.parse_args()
def main(action, **kwargs):
try:
RULES_CHECK_IN_PROGRESS.touch()
except Exception as e:
logger.error("Failed to create RULES_CHECK_IN_PROGRESS file: %s", e)
tls_check.reset()
setup_environment()
Merger.update_merged_config()
loop = asyncio.get_event_loop()
rp, rc = restore_state(RealProtector(), RulesChecker(loop))
action_func = ACTIONS[action]
loop.run_until_complete(action_func(rc, rp, **kwargs))
save_state(rp, rc)
logger.info("Script finished")
if __name__ == "__main__":
args = parse_arguments()
kwargs = {}
if args.refill_ports:
kwargs["refill"] = args.refill_ports
if args.check_all:
kwargs["check_all"] = args.check_all
try:
main(args.action, **kwargs)
except Exception as e:
RULES_CHECK_IN_PROGRESS.unlink(missing_ok=True)
logger.exception("rules checker failed with unhandled error: %r", e)
sys.exit(2)
else:
RULES_CHECK_IN_PROGRESS.unlink(missing_ok=True)