From a96a1a9f492e7de57bc83d875f99a7230eb1f92a Mon Sep 17 00:00:00 2001 From: EEliseeva Date: Sun, 5 May 2024 11:13:21 +0200 Subject: [PATCH 01/10] cti module merged --- .gitignore | 4 +- README.md | 70 +-- commands/CtiController.php | 537 ++++++++++++++++++ commands/NetworkController.php | 40 +- commands/db_retention.py | 16 +- commands/network_model_builder.py | 8 +- commands/rules_downloader.py | 14 +- composer.json | 4 +- deployment/config_templates/cti_config.json | 6 + .../config_templates/secmon_config.yaml | 39 ++ deployment/dockerfiles/secmon_cti.Dockerfile | 6 + .../secmon_db_retention.Dockerfile | 1 + .../m231214_201649_create_cti_tables.php | 79 +++ ...ti_id_columns_to_security_events_table.php | 27 + ...dd_complementary_columns_to_cti_tables.php | 70 +++ models/CtiCrowdsec.php | 15 + models/CtiModel.php | 218 +++++++ models/CtiNerd.php | 15 + models/SecurityEvents.php | 27 +- secmon_deploy.sh | 3 +- secmon_manager.py | 207 ++++--- secmon_preconfig.sh | 5 +- views/security-events/view.php | 522 +++++++++++++++++ web/css/site.css | 24 + 24 files changed, 1769 insertions(+), 188 deletions(-) create mode 100644 commands/CtiController.php create mode 100644 deployment/config_templates/cti_config.json create mode 100644 deployment/config_templates/secmon_config.yaml create mode 100644 deployment/dockerfiles/secmon_cti.Dockerfile create mode 100755 migrations/m231214_201649_create_cti_tables.php create mode 100755 migrations/m231228_190055_add_source_cti_id_and_destination_cti_id_columns_to_security_events_table.php create mode 100644 migrations/m240305_171602_add_complementary_columns_to_cti_tables.php create mode 100644 models/CtiCrowdsec.php create mode 100644 models/CtiModel.php create mode 100644 models/CtiNerd.php diff --git a/.gitignore b/.gitignore index 15829d8b..f7d19c07 100755 --- a/.gitignore +++ b/.gitignore @@ -33,7 +33,9 @@ docker-compose.yml /config/.lock /config/db.php /config/anomaly_config.ini -/config/secmon_config.ini +/config/secmon_config.yaml +/config/cti_config.json +config/ip_rep.csv /config/aggregator_config.ini /deployment/certificates /rules diff --git a/README.md b/README.md index f96543e0..fde884a0 100755 --- a/README.md +++ b/README.md @@ -37,14 +37,8 @@ cd secmon sudo python3 secmon_manager.py deploy # Create password for database user 'secmon' during installation - -# Default login credentials user:secmon, password:password -# !!! Change password after first login !!! -https://:8443/secmon/web ``` -After successful installation configure logs forwarding on clients using [rsyslog service](./README.md#how-to-configure-clients-for-logs-forwarding). - --- ### CentOS 8 @@ -74,14 +68,8 @@ cd secmon sudo python3 secmon_manager.py deploy # Create password for database user 'secmon' during installation - -# Default login credentials user:secmon, password:password -# !!! Change password after first login !!! -https://:8443/secmon/web ``` -After successful installation configure logs forwarding on clients using [rsyslog service](./README.md#how-to-configure-clients-for-logs-forwarding). - --- ### Rocky 9 @@ -112,15 +100,9 @@ cd secmon sudo python3 secmon_manager.py deploy # Create password for database user 'secmon' during installation - -# Default login credentials user:secmon, password:password -# !!! Change password after first login !!! -https://:8443/secmon/web ``` Installation of Docker on Rocky Linux 9: [installation help](./docs/docker_installation_RL9.md). -After successful installation configure logs forwarding on clients using [rsyslog service](./README.md#how-to-configure-clients-for-logs-forwarding). - --- ### Ubuntu 22.04 @@ -131,7 +113,7 @@ sudo apt clean all sudo apt -y update # Install git, firewall & rsyslog -sudo apt install -y git ufw rsyslog +sudo apt install -y git ufw firewalld rsyslog # Install python packages sudo apt-get install -y make build-essential libssl-dev zlib1g-dev \ @@ -143,6 +125,7 @@ sudo tar -xzf Python-3.6.15.tgz cd Python-3.6.15 sudo ./configure --enable-optimizations -with-lto --with-pydebug sudo make altinstall +cd .. # Setting up firewall sudo ufw allow 8080/tcp @@ -157,19 +140,25 @@ cd secmon sudo python3 secmon_manager.py deploy # Create password for database user 'secmon' during installation - -# Default login credentials user:secmon, password:password -# !!! Change password after first login !!! -https://:8443/secmon/web ``` -After successful installation configure logs forwarding on clients using [rsyslog service](./README.md#how-to-configure-clients-for-logs-forwarding). - --- ## How to Use + +### Before first usage +After successful installation configure logs forwarding on clients using [rsyslog service](./README.md#how-to-configure-clients-for-logs-forwarding). + +### Login info +Application URL: `https://:8443/secmon/web` + +Default login credentials: \ +`user: secmon` \ +`password: password`\ +Credentials should be changed after first login! + ### SecMon Manager -SecMon manager (*secmon_manager.py*) is a python script located in root directory of SecMon repository. It is used for managing SecMon services as docker containers. +SecMon manager (`secmon_manager.py`) is a python script located in root directory of SecMon repository. It is used for managing SecMon services as docker containers. ```bash # Show list of all available parameters python3 secmon_manager.py help @@ -183,7 +172,7 @@ python3 secmon_manager.py start # Restart running/stopped SecMon system python3 secmon_manager.py restart -# Remove SecMon enrichment containers +# Remove SecMon containers python3 secmon_manager.py remove # Manually run configuration script @@ -197,13 +186,28 @@ python3 secmon_manager.py update-rules ``` ## Configuration ### Turn on/off enrichment module -Set value `true` /`false` in the file `./config/secmon_config.ini` for a particular enrichment module which you want to turn on/off: -```ini -[ENRICHMENT] -correlation = true -geoip = true -network_model = true +Set value `true` /`false` in the file `./config/secmon_config.yaml` for a particular enrichment module which you want to turn on/off: +```yaml +- name: Network_model + enabled: true + args: [] +- name: correlator + enabled: true + args: [] ``` + +### Pass Docker run arguments to an enrichment module +Add `swith` and a following `argument` if needed in the file `./config/secmon_config.yaml` for a particular enrichment module which you want to modify: +```yaml +- name: CTI + enabled: true + args: + - "-e" + - NERD_API_KEY + - "-e" + - CROWD_API_KEY +``` + After any changes in configuration or rule states, restart the SecMon system with the command: ```bash python3 secmon_manager.py restart diff --git a/commands/CtiController.php b/commands/CtiController.php new file mode 100644 index 00000000..0b754dd8 --- /dev/null +++ b/commands/CtiController.php @@ -0,0 +1,537 @@ +openNonBlockingStream("/var/www/html/secmon/config/aggregator_config.ini"); + $save_to_db = 0; + $module_loaded = false; #variable used for reading line after CTI module in config file + $next_module = "correlator"; + if ($temp_config) { + while (($line = fgets($temp_config)) !== false) { + if ($module_loaded == true) { + $parts = explode(":", $line); + $next_module = strtolower(trim($parts[0])); + $module_loaded = false; + } + + if (strpos($line, "CTI:") !== FALSE) { + $parts = explode(":", $line); + $port = trim($parts[1]); + $module_loaded = true; + } + } + } else { + throw new Exception('Could not open a config file'); + } + + $yaml_secmon = file_get_contents('/var/www/html/secmon/config/secmon_config.yaml'); + $yaml_secmon_data = Yaml::parse($yaml_secmon); + $db_config = $yaml_secmon_data["DATABASE"]; + + if ($db_config) { + if ($db_config["host"] !== FALSE) { + $host = $db_config["host"]; + } + if ($db_config["database"] !== FALSE) { + $database = $db_config["database"]; + } + if ($db_config["user"] !== FALSE) { + $user = $db_config["user"]; + } + if ($db_config["password"] !== FALSE) { + $password = $db_config["password"]; + } + } else { + throw new Exception('Not all arguments were specified'); + } + + fclose($temp_config); + $temp_config = escapeshellarg("/var/www/html/secmon/config/aggregator_config.ini"); + $last_line = `tail -n 1 $temp_config`; #get last line of temp file + + if (strpos($last_line, "CTI:") !== FALSE) { + $save_to_db = 1; + } + + if (!is_numeric($port)) { + throw new Exception('One of ports is not a numeric value'); + } + + $zmq = new ZMQContext(); + $recSocket = $zmq->getSocket(ZMQ::SOCKET_PULL); + $recSocket->bind("tcp://*:" . $port); + + $sendSocket = $zmq->getSocket(ZMQ::SOCKET_PUSH); + $sendSocket->connect("tcp://secmon_" . $next_module . ":" . $port); + + $nerd_auth = getenv("NERD_API_KEY"); + if (empty($nerd_auth)) { + Yii::info("No NERD API authorization key" . PHP_EOL); + } + $crowd_auth = getenv("CROWD_API_KEY"); + if (empty($crowd_auth)) { + Yii::info("No CrowdSec API authorization key" . PHP_EOL); + } + + date_default_timezone_set("Europe/Bratislava"); + echo "[" . date("Y-m-d H:i:s") . "] CTI module started!" . PHP_EOL; + + $client = new Client(['responseConfig' => [ + 'format' => Client::FORMAT_JSON + ]]); + + $json_cti = file_get_contents('/var/www/html/secmon/config/cti_config.json'); + $json_cti_data = json_decode($json_cti, true); + $api_time_validity = $json_cti_data["api_validity"]; + $file_time_validity = $json_cti_data["file_validity"]; + $whitelist = $json_cti_data["whitelist"]; + + while (true) { + $srcIp = $dstIp = -1; + $msg = $recSocket->recv(ZMQ::MODE_NOBLOCK); + + if (empty($msg)) { + usleep(30000); + } else { + //print("GOT SOME MESSAGE:\n"); + $position1 = strpos($msg, "src="); + if ($position1 != FALSE) { + $position2 = strpos($msg, " ", $position1); + $position3 = $position2 - $position1 - strlen("src="); + $srcIp = substr($msg, $position1 + strlen("src="), $position3); + } + + $position1 = strpos($msg, "dst="); + if ($position1 != FALSE) { + $position2 = strpos($msg, " ", $position1); + $position3 = $position2 - $position1 - strlen("dst="); + $dstIp = substr($msg, $position1 + strlen("dst="), $position3); + } + $connection = pg_connect("host=" . $host . " dbname=" . $database . " user=" . $user . " password=" . $password); + + if ($srcIp != -1) { + //print("PROCESSING SRC\n"); + $src_cti_id = $this->processIp($srcIp, $whitelist, $connection, $client, $nerd_auth, $crowd_auth, $api_time_validity, $file_time_validity); + $msg = str_replace("\n", "", $msg); + $msg = $msg . " src_cti_id=" . strval($src_cti_id); + } + + if ($dstIp != -1) { + //print("PROCESSING DST\n"); + $dst_cti_id = $this->processIp($dstIp, $whitelist, $connection, $client, $nerd_auth, $crowd_auth, $api_time_validity, $file_time_validity); + $msg = $msg . " dst_cti_id=" . strval($dst_cti_id) . " "; + } + + //print("FINAL MESSAGE:\n"); + pg_close($connection); + + if ($save_to_db) { + //print("SAVING TO DB\n"); + $event = SecurityEvents::extractCefFields($msg, 'normalized'); + if ($event->save()) { + $sendSocket->send($event->id . ':' . $msg, ZMQ::MODE_NOBLOCK); + } + } else { + //print("SOMEONE ELSE SAVING TO DB\n"); + $sendSocket->send($msg, ZMQ::MODE_NOBLOCK); + } + } + } + } + + function processIp($ip, $whitelist, $connection, $client, $nerd_auth, $crowd_auth, $api_time_validity, $file_time_validity) + { + if (in_array($ip, $whitelist)) { + //print($ip . "is in whitelist"); + return "0"; + } + + if ($connection) { + //print("Successfuly connected to DB\n"); + $main = $this->selectFromPairingTable($ip, $connection); + if ($main == null) { + //print("IP was not recorded before\n"); + $main[0] = $this->recordToPairingTable($ip, $connection); + } + + ############# NERD ############# + if ($main[2] != null) { + //print("Pairing table has NERD table linked\n"); + $object = $this->selectFromNERDTable($main[2], $connection, $ip, $client, $nerd_auth, $api_time_validity); + } else { + //print("Pairing table hasn't NERD table linked\n"); + $object = $this->updateFromNERDapi($ip, $client, $nerd_auth); + if ($object != null && $object != -1) { + $nerd_id = $this->recordToNERDTable($object, $connection); + $this->updatePairingTableNERD($connection, $main[0], $nerd_id); + $main[2] = $nerd_id; + } + } + + ############# CROWD ############# + if ($main[1] != null) { + //print("Pairing table has CROWD table linked\n"); + $object = $this->selectFromCROWDTable($main[1], $connection, $ip, $client, $crowd_auth, $api_time_validity); + } else { + //print("Pairing table hasn't CROWD table linked\n"); + $object = $this->updateFromCROWDapi($ip, $client, $crowd_auth); + if ($object != null && $object != -1) { + $crowd_id = $this->recordToCROWDTable($object, $connection); + $this->updatePairingTableCROWD($connection, $main[0], $crowd_id); + $main[1] = $crowd_id; + } + } + } else { + throw new Exception("Error while connecting to database!". PHP_EOL); + } + + return $main[0]; + } + + function selectFromPairingTable($ip, $connection) + { + //print("selectFromPairingTable started\n"); + if ($ip != -1) { + //print("IP is defined\n"); + $result = pg_query_params($connection, 'SELECT id, fk_crowdsec_id, fk_nerd_id from cti where ip = $1', array($ip)); + if (pg_num_rows($result) > 0) { + //print("Found IP in DB\n"); + $row = pg_fetch_row($result); + return $row; + } + + //print("No IP in DB\n"); + return null; + } + } + + function recordToPairingTable($ip, $connection) + { + //print("recordToPairingTable started\n"); + if ($ip != -1) { + //print("IP is defined\n"); + $result = pg_query_params($connection, 'INSERT INTO cti(ip) VALUES ($1) RETURNING id', array($ip)); + $id = pg_fetch_row($result)[0]; + if ($result == false) { + //print("Query failed\n"); + return -1; + } + //print("Successfuly inserted\n"); + return $id; + } + } + + function selectFromNERDTable($id, $connection, $ip, $client, $nerd_auth, $api_time_validity) + { + //print("selectFromNERDTable started\n"); + if ($id != -1) { + //print("ID is defined\n"); + $result = pg_query_params($connection, 'SELECT * from cti_nerd where id = $1', array($id)); + if (pg_num_rows($result) > 0) { + //print("Found IP in DB\n"); + $row = pg_fetch_row($result); + + $object = new \stdClass(); + $object->nerd_timestamp = date('Y-m-d H:i:s', strtotime((string)$row[5])); + // flag = 0, selected from DB, no update on main table needed + $object->nerd = 0; + + $now = new \DateTime(date('Y-m-d H:i:s', strtotime('now'))); + $stamp = new \DateTime($object->nerd_timestamp); + $interval = $now->diff($stamp); + $hours = $interval->h + ($interval->days * 24); + + if ($hours > $api_time_validity) { + //print("Record is old enough, needs refresh\n"); + $object = $this->updateFromNERDapi($ip, $client, $nerd_auth); + if ($object) { + $this->updateNERDTable($id, $object, $connection); + } + } + } + } + return $object; + } + + function updateNERDTable($id, $object, $connection) + { + //print("updateNERDTable started\n"); + $result = pg_query_params( + $connection, + 'UPDATE cti_nerd SET + fmp=$1, blacklists=$2, rep=$3, last_checked_at=$4, + as_id=$5, as_name=$6, ip_range=$7, ip_range_rep=$8, + events=$9, geo_city=$10, geo_country=$11, + hostname=$12, last_activity=$13, first_activity=$14 + WHERE id = $15', + array( + $object->fmp, $object->blacklists, $object->rep, $object->nerd_timestamp, + $object->nerd_AS_id, $object->nerd_AS_name, $object->nerd_ip_range, $object->nerd_ip_range_rep, + $object->events, $object->nerd_city, $object->nerd_country, + $object->nerd_hostname, $object->nerd_last_activity, $object->nerd_first_activity, + $id + ) + ); + $id = pg_fetch_row($result)[0]; + if ($result == false) { + //print("Query failed\n"); + } + //print("Successfuly updated\n"); + } + + function updateFromNERDapi($ip, $client, $nerd_auth) + { + //print("updateFromNERDapi started\n"); + $nerd_response = $client->createRequest() + ->setMethod('GET') + ->setUrl('https://nerd.cesnet.cz/nerd/api/v1/ip/' . (string)$ip . '/full') + ->addHeaders(['Authorization' => $nerd_auth]) + ->send(); + if ($nerd_response->statusCode == 404) { + //print("No NERD record found\n"); + return -1; + } elseif ($nerd_response->statusCode != 200) { + //print("NERD API call failed\n"); + return null; + } + + $object = new \stdClass(); + $object->fmp = $nerd_response->data["fmp"]["general"]; + $bl = array(); + foreach ($nerd_response->data["bl"] as $list) { + array_push($bl, $list["name"]); + } + $object->blacklists = implode(", ", $bl); + $object->rep = $nerd_response->data["rep"]; + $object->nerd_AS_id = $nerd_response->data["asn"][0]["_id"]; + $object->nerd_AS_name = $nerd_response->data["asn"][0]["name"]; + $object->nerd_ip_range = $nerd_response->data["bgppref"]["_id"]; + $object->nerd_ip_range_rep = $nerd_response->data["bgppref"]["rep"]; + + $events_cat = array(); + foreach ($nerd_response->data["events"] as $list) { + array_push($events_cat, $list["cat"]); + } + $object->events = implode(", ", array_unique($events_cat)); + $object->nerd_city = $nerd_response->data["geo"]["city"]; + $object->nerd_country = $nerd_response->data["geo"]["ctry"]; + $object->nerd_hostname = $nerd_response->data["hostname"]; + $object->nerd_last_activity = $nerd_response->data["last_activity"]; + $object->nerd_first_activity = $nerd_response->data["ts_added"]; + + $object->nerd_timestamp = date('Y-m-d H:i:s', strtotime('now')); + // flag = 1, update on main table needed, new data from API + $object->nerd = 1; + return $object; + } + + function recordToNERDTable($object, $connection) + { + //print("recordToNERDTable started\n"); + $result = pg_query_params( + $connection, + 'INSERT INTO cti_nerd(fmp, blacklists, rep, last_checked_at, + as_id, as_name, ip_range, ip_range_rep, + events, geo_city, geo_country, + hostname, last_activity, first_activity) + VALUES ($1, $2, $3, $4, + $5, $6, $7, $8, + $9, $10, $11, + $12, $13, $14) RETURNING id', + array( + $object->fmp, $object->blacklists, $object->rep, $object->nerd_timestamp, + $object->nerd_AS_id, $object->nerd_AS_name, $object->nerd_ip_range, $object->nerd_ip_range_rep, + $object->events, $object->nerd_city, $object->nerd_country, + $object->nerd_hostname, $object->nerd_last_activity, $object->nerd_first_activity + ) + ); + $id = pg_fetch_row($result)[0]; + if ($result == false) { + //print("Query failed\n"); + return -1; + } + return $id; + } + + function updatePairingTableNERD($connection, $main_id, $nerd_id) + { + //print("updatePairingTableNERD started\n"); + $result = pg_query_params($connection, 'UPDATE cti SET fk_nerd_id = $1 WHERE id = $2 RETURNING id', array($nerd_id, $main_id)); + $id = pg_fetch_row($result)[0]; + if ($result == false) { + //print("Query failed\n"); + } + } + + function selectFromCROWDTable($id, $connection, $ip, $client, $crowd_auth, $api_time_validity) + { + //print("selectFromCROWDTable started\n"); + if ($id != -1) { + //print("ID is defined\n"); + $result = pg_query_params($connection, 'SELECT * from cti_crowdsec where id = $1', array($id)); + if (pg_num_rows($result) > 0) { + //print("Found IP in DB\n"); + $row = pg_fetch_row($result); + + $object = new \stdClass(); + $object->crowd_timestamp = date('Y-m-d H:i:s', strtotime((string)$row[5])); + // flag = 0, selected from DB, no update on main table needed + $object->crowd = 0; + + $now = new \DateTime(date('Y-m-d H:i:s', strtotime('now'))); + $stamp = new \DateTime($object->crowd_timestamp); + $interval = $now->diff($stamp); + $hours = $interval->h + ($interval->days * 24); + + if ($hours > $api_time_validity) { + //print("Record is old enough, needs refresh\n"); + $object = $this->updateFromCROWDapi($ip, $client, $crowd_auth); + if ($object) { + $this->updateCROWDTable($id, $object, $connection); + } + } + } + } + return $object; + } + + function updateCROWDTable($id, $object, $connection) + { + //print("updateCROWDTable started\n"); + $result = pg_query_params( + $connection, + 'UPDATE cti_crowdsec SET + behavior=$1, classification=$2, score_overall=$3, last_checked_at=$4, + as_num=$5, as_name=$6, ip_range_24=$7, + ip_range_24_rep=$8, geo_city=$9, geo_country=$10, + reverse_dns=$11, last_seen=$12, first_seen=$13 + WHERE id = $14', + array( + $object->behavior, $object->classification, $object->score_overall, $object->crowd_timestamp, + $object->crowd_AS_id, $object->crowd_AS_name, $object->crowd_ip_range, + $object->crowd_ip_range_rep, $object->crowd_city, $object->crowd_country, + $object->crowd_reverse_dns, $object->crowd_last_seen, $object->crowd_first_seen, + $id + ) + ); + $id = pg_fetch_row($result)[0]; + if ($result == false) { + //print("Query failed\n"); + } + } + + + function updateFromCROWDapi($ip, $client, $crowd_auth) + { + //print("updateFromCROWDapi started\n"); + $crowd_response = $client->createRequest() + ->setMethod('GET') + ->setUrl('https://cti.api.crowdsec.net/v2/smoke/' . (string)$ip) + ->addHeaders(['x-api-key' => $crowd_auth]) + ->send(); + if ($crowd_response->statusCode == 404) { + //print("No CROWD record found\n"); + return -1; + } elseif ($crowd_response->statusCode != 200) { + //print("CROWD API call failed\n"); + return null; + } + + $object = new \stdClass(); + $beh = array(); + foreach ($crowd_response->data["behaviors"] as $list) { + array_push($beh, $list["label"]); + } + $object->behavior = implode(", ", $beh); + $class = array(); + foreach ($crowd_response->data["classifications"]["classifications"] as $list) { + array_push($class, $list["label"]); + } + $object->classification = implode(", ", $class); + $object->score_overall = $crowd_response->data["scores"]["overall"]["total"]; + $object->crowd_AS_id = $crowd_response->data["as_num"]; + $object->crowd_AS_name = $crowd_response->data["as_name"]; + $object->crowd_ip_range = $crowd_response->data["ip_range_24"]; + $object->crowd_ip_range_rep = $crowd_response->data["ip_range_24_reputation"] . " " . $crowd_response->data["ip_range_24_score"]; + $object->crowd_city = $crowd_response->data["location"]["city"]; + $object->crowd_country = $crowd_response->data["location"]["country"]; + $object->crowd_reverse_dns = $crowd_response->data["reverse_dns"]; + $object->crowd_last_seen = $crowd_response->data["history"]["last_seen"]; + $object->crowd_first_seen = $crowd_response->data["history"]["first_seen"]; + $object->crowd_timestamp = date('Y-m-d H:i:s', strtotime('now')); + // flag = 1, update on main table needed, new data from API + $object->crowd = 1; + return $object; + } + + function recordToCROWDTable($object, $connection) + { + //print("recordToCROWDTable started\n"); + $result = pg_query_params( + $connection, + 'INSERT INTO cti_crowdsec(behavior, classification, score_overall, last_checked_at, + as_num, as_name, ip_range_24, + ip_range_24_rep, geo_city, geo_country, + reverse_dns, last_seen, first_seen) + VALUES ($1, $2, $3, $4, + $5, $6, $7, + $8, $9, $10, + $11, $12, $13) + RETURNING id', + array( + $object->behavior, $object->classification, $object->score_overall, $object->crowd_timestamp, + $object->crowd_AS_id, $object->crowd_AS_name, $object->crowd_ip_range, + $object->crowd_ip_range_rep, $object->crowd_city, $object->crowd_country, + $object->crowd_reverse_dns, $object->crowd_last_seen, $object->crowd_first_seen + ) + ); + $id = pg_fetch_row($result)[0]; + if ($result == false) { + //print("Query failed\n"); + return -1; + } + return $id; + } + + function updatePairingTableCROWD($connection, $main_id, $crowd_id) + { + //print("updatePairingTableCROWD started\n"); + $result = pg_query_params($connection, 'UPDATE cti SET fk_crowdsec_id = $1 WHERE id = $2 RETURNING id', array($crowd_id, $main_id)); + $id = pg_fetch_row($result)[0]; + if ($result == false) { + //print("Query failed\n"); + } + } + + function openNonBlockingStream($file) + { + $stream = fopen($file, 'r+'); + + if ($stream === false) { + return null; + } + + stream_set_blocking($stream, false); + + return $stream; + } +} diff --git a/commands/NetworkController.php b/commands/NetworkController.php index 516f19e2..456bf856 100755 --- a/commands/NetworkController.php +++ b/commands/NetworkController.php @@ -9,6 +9,7 @@ use ZMQ; use ZMQContext; use ZMQSocketException; +use Symfony\Component\Yaml\Yaml; require '/var/www/html/secmon/vendor/autoload.php'; @@ -39,32 +40,27 @@ public function actionIndex(){ throw new Exception('Could not open a config file'); } - $middleware_config_file = $this->openNonBlockingStream("/var/www/html/secmon/config/secmon_config.ini"); - if($middleware_config_file){ - while(($line = fgets($middleware_config_file)) !== false){ - if(strpos($line, "host =") !== FALSE){ - $parts = explode("=", $line); - $host = trim($parts[1]); - } - if(strpos($line, "database =") !== FALSE){ - $parts = explode("=", $line); - $database = trim($parts[1]); - } - if(strpos($line, "user =") !== FALSE){ - $parts = explode("=", $line); - $user = trim($parts[1]); - } - if(strpos($line, "password =") !== FALSE){ - $parts = explode("=", $line); - $password = trim($parts[1]); - } + $yaml_secmon = file_get_contents('/var/www/html/secmon/config/secmon_config.yaml'); + $yaml_secmon_data = Yaml::parse($yaml_secmon); + $db_config = $yaml_secmon_data["DATABASE"]; + if ($db_config) { + if ($db_config["host"] !== FALSE) { + $host = $db_config["host"]; } - }else{ - throw new Exception('Not all arguments were specified'); + if ($db_config["database"] !== FALSE) { + $database = $db_config["database"]; + } + if ($db_config["user"] !== FALSE) { + $user = $db_config["user"]; + } + if ($db_config["password"] !== FALSE) { + $password = $db_config["password"]; + } + } else { + throw new Exception('Not all arguments were specified'); } fclose($aggregator_config_file); - fclose($middleware_config_file); $aggregator_config_file = escapeshellarg("/var/www/html/secmon/config/aggregator_config.ini"); $last_line = `tail -n 1 $aggregator_config_file`; #get last line of temp file diff --git a/commands/db_retention.py b/commands/db_retention.py index a611399a..d8ff5f47 100755 --- a/commands/db_retention.py +++ b/commands/db_retention.py @@ -1,11 +1,11 @@ #!/usr/bin/env python3 # encoding: utf-8 -import configparser import psycopg2 import os import time import datetime +import yaml def wait_for_db(): while(is_db_ready() is not True): @@ -24,7 +24,7 @@ def is_db_ready(): def connect(): conn = False try: - conn = psycopg2.connect(host=config.get('DATABASE', 'host'),database=config.get('DATABASE', 'database'), user=config.get('DATABASE', 'user'), password=config.get('DATABASE', 'password')) + conn = psycopg2.connect(host=config['DATABASE']['host'],database=config['DATABASE']['database'], user=config['DATABASE']['user'], password=config['DATABASE']['password']) except: os.system('echo -e "Connection to the database was unsuccessful"') return False @@ -34,7 +34,7 @@ def size_check(max_db_size): os.system('echo -e "Proceeding database size check"') connection = connect() cursor = connection.cursor() - querry = "SELECT pg_size_pretty(pg_database_size(\'" + config.get('DATABASE', 'database') + "\'));" + querry = "SELECT pg_size_pretty(pg_database_size(\'" + config['DATABASE']['database'] + "\'));" cursor.execute(querry) db_size = cursor.fetchone() act_size = db_size[0].split() @@ -61,12 +61,12 @@ def timestamp_check(last_date): connection.close() #read configuration file -config = configparser.ConfigParser() -config.read('./config/secmon_config.ini') +secmon_config_file = open('./config/secmon_config.yaml') +config = yaml.safe_load(secmon_config_file) -max_db_size = config.get('DATABASE', 'max_size') -no_of_days = config.get('DATABASE', 'max_days') -sleep_interval= config.get('DATABASE', 'sleep_interval') +max_db_size = config['DATABASE']['max_size'] +no_of_days = config['DATABASE']['max_days'] +sleep_interval= config['DATABASE']['sleep_interval'] wait_for_db() diff --git a/commands/network_model_builder.py b/commands/network_model_builder.py index 0e3df669..a4fb9464 100755 --- a/commands/network_model_builder.py +++ b/commands/network_model_builder.py @@ -5,15 +5,15 @@ # sudo yum install nmap # sudo pip3 install python-libnmap -import configparser import psycopg2 import sys import os +import yaml from libnmap.parser import NmapParser def connect(): try: - conn = psycopg2.connect(host=config.get('DATABASE', 'host'),database=config.get('DATABASE', 'database'), user=config.get('DATABASE', 'user'), password=config.get('DATABASE', 'password')) + conn = psycopg2.connect(host=config['DATABASE']['host'],database=config['DATABASE']['database'], user=config['DATABASE']['user'], password=config['DATABASE']['password']) except: print ("I am unable to connect to the database") return conn @@ -139,8 +139,8 @@ def insert(host): p = NmapParser.parse_fromfile(sys.argv[1]) #read configuration file -config = configparser.ConfigParser() -config.read('/var/www/html/secmon/config/secmon_config.ini') +secmon_config_file = open('/var/www/html/secmon/config/secmon_config.yaml') +config = yaml.safe_load(secmon_config_file) for host in p.hosts: if already_exists(host.address): diff --git a/commands/rules_downloader.py b/commands/rules_downloader.py index d835b127..871a3e36 100755 --- a/commands/rules_downloader.py +++ b/commands/rules_downloader.py @@ -1,17 +1,17 @@ #!/usr/bin/env python3 # encoding: utf-8 -import configparser import os import sys +import yaml enviroment = sys.argv[1] # supported values: web, os if enviroment == "os": # Execute this if rules update called in deployment - config = configparser.ConfigParser() - config.read('./config/secmon_config.ini') - repo_url = config.get('RULES_REPOSITORY', 'url') + yaml_config = open('./config/secmon_config.yaml') + config = yaml.safe_load(yaml_config) + repo_url = config['RULES_REPOSITORY']['url'] os.system(f'git clone {repo_url} ../.git/temp/rules_repository') os.system(f'mv ../.git/temp/rules_repository/normalization/* ./rules/normalization/available/') @@ -22,9 +22,9 @@ elif enviroment == 'web': # Execute this if rules update called from web rules = sys.argv[2] # supported values: correlation, normalization - config = configparser.ConfigParser() - config.read('/var/www/html/secmon/config/secmon_config.ini') - repo_url = config.get('RULES_REPOSITORY', 'url') + yaml_config = open('/var/www/html/secmon/config/secmon_config.yaml') + config = yaml.safe_load(yaml_config) + repo_url = config['RULES_REPOSITORY']['url'] os.system(f'git clone {repo_url} ./assets/temp/rules_repository') if rules == 'correlation': diff --git a/composer.json b/composer.json index 5f61b096..2c44dae1 100755 --- a/composer.json +++ b/composer.json @@ -34,7 +34,9 @@ "ext-json": "*", "geoip2/geoip2": "~2.0", "beaten-sect0r/yii2-flatpickr": "*", - "onmotion/yii2-widget-apexcharts": "1.0.7" + "onmotion/yii2-widget-apexcharts": "1.0.7", + "yiisoft/yii2-httpclient": "*", + "symfony/yaml": "5.4.*" }, "require-dev": { "codeception/codeception": "*", diff --git a/deployment/config_templates/cti_config.json b/deployment/config_templates/cti_config.json new file mode 100644 index 00000000..324ed634 --- /dev/null +++ b/deployment/config_templates/cti_config.json @@ -0,0 +1,6 @@ +{ + "whitelist": [ + ], + "api_validity": 24, + "file_validity": 24 +} \ No newline at end of file diff --git a/deployment/config_templates/secmon_config.yaml b/deployment/config_templates/secmon_config.yaml new file mode 100644 index 00000000..00b61ccd --- /dev/null +++ b/deployment/config_templates/secmon_config.yaml @@ -0,0 +1,39 @@ +--- +DATABASE: + database: secmon + user: secmon + password: + max_size: 10000000 + max_days: 30 + sleep_interval: 600 + host: secmon_db +DEVICE: + name: secmon + log_input: "/var/log/secmon" +NORMALIZATION: + input_NP: "/var/log/secmon/__secOutput" + output_NP: "/var/log/secmon/__secInput" +CORRELATION: + input_NP: "/var/www/html/secmon/__secOutput" + output_NP: "/var/www/html/secmon/__secInput" +RULES_REPOSITORY: + url: https://github.com/jlasti/secmon-rules.git +ENRICHMENT: +- name: Geoip + enabled: true + args: [] +- name: CTI + enabled: true + args: + - "-e" + - NERD_API_KEY + #- NERD_API_KEY="INSERT_YOUR_KEY_THERE" + - "-e" + - CROWD_API_KEY + #- CROWD_API_KEY="INSERT_YOUR_KEY_THERE" +- name: Network_model + enabled: true + args: [] +- name: correlator + enabled: true + args: [] diff --git a/deployment/dockerfiles/secmon_cti.Dockerfile b/deployment/dockerfiles/secmon_cti.Dockerfile new file mode 100644 index 00000000..ae2f2c8f --- /dev/null +++ b/deployment/dockerfiles/secmon_cti.Dockerfile @@ -0,0 +1,6 @@ +FROM secmon_base + +# Set working directory +WORKDIR /var/www/html/secmon + +ENTRYPOINT ["sh", "-c", "./yii cti"] diff --git a/deployment/dockerfiles/secmon_db_retention.Dockerfile b/deployment/dockerfiles/secmon_db_retention.Dockerfile index 88a30c91..94d15693 100644 --- a/deployment/dockerfiles/secmon_db_retention.Dockerfile +++ b/deployment/dockerfiles/secmon_db_retention.Dockerfile @@ -4,6 +4,7 @@ FROM python:3.9-alpine RUN pip3 install --upgrade pip RUN pip3 install psycopg2-binary RUN pip3 install -U configparser +RUN pip3 install -U pyyaml # Set working directory WORKDIR /home/secmon diff --git a/migrations/m231214_201649_create_cti_tables.php b/migrations/m231214_201649_create_cti_tables.php new file mode 100755 index 00000000..a26e1274 --- /dev/null +++ b/migrations/m231214_201649_create_cti_tables.php @@ -0,0 +1,79 @@ +createTable('cti_crowdsec', [ + 'id' => $this->primaryKey(), + 'first_seen' => $this->string(), + 'last_seen' => $this->string(), + 'behavior' => $this->string(), + 'false_pos' => $this->string(), + 'classification' => $this->string(), + 'score_overall' => $this->integer(), + 'last_checked_at' => $this->timestamp(), + ]); + + // Create cti_nerd table + $this->createTable('cti_nerd', [ + 'id' => $this->primaryKey(), + 'asn_name' => $this->string(), + 'asn_rep' => $this->float(), + 'fmp' => $this->float(), + 'blacklists' => $this->string(), + 'rep' => $this->float(), + 'last_checked_at' => $this->timestamp(), + ]); + + // Create cti table with foreign keys + $this->createTable('cti', [ + 'id' => $this->primaryKey(), + 'fk_crowdsec_id' => $this->integer(), + 'fk_nerd_id' => $this->integer(), + 'ip' => $this->string(), + ]); + + // Add foreign keys + $this->addForeignKey('fk_cti_crowdsec', 'cti', 'fk_crowdsec_id', 'cti_crowdsec', 'id', 'CASCADE', 'CASCADE'); + $this->addForeignKey('fk_cti_nerd', 'cti', 'fk_nerd_id', 'cti_nerd', 'id', 'CASCADE', 'CASCADE'); + + } + + /** + * {@inheritdoc} + */ + public function safeDown() + { + // Drop tables in reverse order + $this->dropForeignKey('fk_cti_nerd', 'cti'); + $this->dropForeignKey('fk_cti_crowdsec', 'cti'); + $this->dropTable('cti'); + $this->dropTable('cti_nerd'); + $this->dropTable('cti_crowdsec'); + } + + /* + // Use up()/down() to run migration code without a transaction. + public function up() + { + + } + + public function down() + { + echo "m231214_201649_create_cti_tables cannot be reverted.\n"; + + return false; + } + */ +} diff --git a/migrations/m231228_190055_add_source_cti_id_and_destination_cti_id_columns_to_security_events_table.php b/migrations/m231228_190055_add_source_cti_id_and_destination_cti_id_columns_to_security_events_table.php new file mode 100755 index 00000000..93d919b5 --- /dev/null +++ b/migrations/m231228_190055_add_source_cti_id_and_destination_cti_id_columns_to_security_events_table.php @@ -0,0 +1,27 @@ +addColumn('security_events', 'source_cti_id', $this->integer()); + $this->addColumn('security_events', 'destination_cti_id', $this->integer()); + } + + /** + * {@inheritdoc} + */ + public function safeDown() + { + $this->dropColumn('security_events', 'source_cti_id'); + $this->dropColumn('security_events', 'destination_cti_id'); + } +} diff --git a/migrations/m240305_171602_add_complementary_columns_to_cti_tables.php b/migrations/m240305_171602_add_complementary_columns_to_cti_tables.php new file mode 100644 index 00000000..52012899 --- /dev/null +++ b/migrations/m240305_171602_add_complementary_columns_to_cti_tables.php @@ -0,0 +1,70 @@ +renameColumn('cti_nerd', 'asn_name', 'as_name'); + $this->dropColumn('cti_nerd', 'asn_rep'); + $this->addColumn('cti_nerd', 'as_id', $this->integer()); + $this->addColumn('cti_nerd', 'ip_range', $this->string()); + $this->addColumn('cti_nerd', 'ip_range_rep', $this->float()); + $this->addColumn('cti_nerd', 'events', $this->string()); + $this->addColumn('cti_nerd', 'geo_city', $this->string()); + $this->addColumn('cti_nerd', 'geo_country', $this->string()); + $this->addColumn('cti_nerd', 'hostname', $this->string()); + $this->addColumn('cti_nerd', 'last_activity', $this->timestamp()); + $this->addColumn('cti_nerd', 'first_activity', $this->timestamp()); + + // Alter existing columns of cti_crowdsec - add new, update unused + $this->addColumn('cti_crowdsec', 'as_num', $this->integer()); + $this->addColumn('cti_crowdsec', 'as_name', $this->string()); + $this->addColumn('cti_crowdsec', 'ip_range_24', $this->string()); + $this->addColumn('cti_crowdsec', 'ip_range_24_rep', $this->string()); + $this->addColumn('cti_crowdsec', 'geo_city', $this->string()); + $this->addColumn('cti_crowdsec', 'geo_country', $this->string()); + $this->addColumn('cti_crowdsec', 'reverse_dns', $this->string()); + $this->dropColumn('cti_crowdsec', 'first_seen'); + $this->dropColumn('cti_crowdsec', 'last_seen'); + $this->addColumn('cti_crowdsec', 'first_seen', $this->timestamp()); + $this->addColumn('cti_crowdsec', 'last_seen', $this->timestamp()); + + } + + /** + * {@inheritdoc} + */ + public function safeDown() + { + $this->renameColumn('cti_nerd', 'as_name', 'asn_name'); + $this->addColumn('cti_nerd', 'asn_rep', $this->float()); + $this->dropColumn('cti_nerd', 'as_id'); + $this->dropColumn('cti_nerd', 'ip_range'); + $this->dropColumn('cti_nerd', 'ip_range_rep'); + $this->dropColumn('cti_nerd', 'events'); + $this->dropColumn('cti_nerd', 'geo_city'); + $this->dropColumn('cti_nerd', 'geo_country'); + $this->dropColumn('cti_nerd', 'hostname'); + $this->dropColumn('cti_nerd', 'last_activity'); + $this->dropColumn('cti_nerd', 'first_activity'); + + $this->dropColumn('cti_crowdsec', 'as_num',); + $this->dropColumn('cti_crowdsec', 'as_name'); + $this->dropColumn('cti_crowdsec', 'ip_range_24'); + $this->dropColumn('cti_crowdsec', 'ip_range_24_rep'); + $this->dropColumn('cti_crowdsec', 'geo_city'); + $this->dropColumn('cti_crowdsec', 'geo_country'); + $this->dropColumn('cti_crowdsec', 'reverse_dns'); + $this->alterColumn('cti_crowdsec', 'first_seen', 'string'); + $this->alterColumn('cti_crowdsec', 'last_seen', 'string'); + } +} diff --git a/models/CtiCrowdsec.php b/models/CtiCrowdsec.php new file mode 100644 index 00000000..9dad349e --- /dev/null +++ b/models/CtiCrowdsec.php @@ -0,0 +1,15 @@ +hasOne(CtiCrowdsec::className(), ['id' => 'fk_crowdsec_id']); + } + + public function getNerd() + { + return $this->hasOne(CtiNerd::className(), ['id' => 'fk_nerd_id']); + } + + public function rules() + { + return [ + [['fk_crowdsec_id', 'fk_nerd_id', 'ip'], 'required'], + [['fk_crowdsec_id', 'fk_nerd_id'], 'integer'], + [['ip'], 'string', 'max' => 255], + ]; + } + + public static function columns() + { + return [ + 'cti.id', + 'cti.fk_crowdsec_id', + 'cti.fk_nerd_id', + 'cti.ip', + 'crowdsec.id', + 'crowdsec.first_seen', + 'crowdsec.last_seen', + 'crowdsec.behavior', + 'crowdsec.false_pos', + 'crowdsec.classification', + 'crowdsec.score_overall', + 'crowdsec.as_num', + 'crowdsec.as_name', + 'crowdsec.ip_range_24', + 'crowdsec.ip_range_24_rep', + 'crowdsec.geo_city', + 'crowdsec.geo_country', + 'crowdsec.reverse_dns', + 'crowdsec.last_checked_at', + 'nerd.id', + 'nerd.as_name', + 'nerd.as_id', + 'nerd.ip_range', + 'nerd.ip_range_rep', + 'nerd.events', + 'nerd.geo_city', + 'nerd.geo_country', + 'nerd.hostname', + 'nerd.first_activity', + 'nerd.last_activity', + 'nerd.fmp', + 'nerd.blacklists', + 'nerd.rep', + 'nerd.last_checked_at', + ]; + } + + public static function getCtiInfo($id) + { + $cti_row = CtiModel::find()->joinWith('nerd')->joinWith('crowdsec')->where(['cti.id' => $id])->one(); + if ($cti_row == null) { + return new CtiModel(); + } + + $cti_row->reputation = isset($cti_row['nerd']) + ? $cti_row['nerd']['rep'] + : CsvService::getRepFromCsv($cti_row['ip']); + + $cti_row->as_name = (object)["nerd" => $cti_row['nerd']['as_name'] ?? null, 'crowd' => $cti_row['crowdsec']['as_name'] ?? null]; + $cti_row->as_num = (object)["nerd" => $cti_row['nerd']['as_id'] ?? null, 'crowd' => $cti_row['crowdsec']['as_num'] ?? null]; + $cti_row->ip_range = (object)["nerd" => $cti_row['nerd']['ip_range'] ?? null, 'crowd' => $cti_row['crowdsec']['ip_range_24'] ?? null]; + $cti_row->ip_range_rep = (object)["nerd" => $cti_row['nerd']['ip_range_rep'] ?? null, 'crowd' => $cti_row['crowdsec']['ip_range_24_rep'] ?? null]; + $cti_row->events = (object)["nerd" => $cti_row['nerd']['events'] ?? null, 'crowd' => $cti_row['crowdsec']['behavior'] ?? null]; + $cti_row->city = (object)["nerd" => $cti_row['nerd']['geo_city'] ?? null, 'crowd' => $cti_row['crowdsec']['geo_city'] ?? null]; + $cti_row->country = (object)["nerd" => $cti_row['nerd']['geo_country'] ?? null, 'crowd' => $cti_row['crowdsec']['geo_country'] ?? null]; + $cti_row->hostname = (object)["nerd" => $cti_row['nerd']['hostname'] ?? null, 'crowd' => $cti_row['crowdsec']['reverse_dns'] ?? null]; + $cti_row->first_seen = (object)["nerd" => $cti_row['nerd']['first_activity'] ?? null, 'crowd' => $cti_row['crowdsec']['first_seen'] ?? null]; + $cti_row->last_seen = (object)["nerd" => $cti_row['nerd']['last_activity'] ?? null, 'crowd' => $cti_row['crowdsec']['last_seen'] ?? null]; + + return $cti_row; + } + + public static function labels() + { + return [ + 'cti.id' => 'ID', + 'cti.fk_crowdsec_id' => 'Crowdsec ID', + 'cti.fk_nerd_id' => 'Nerd ID', + 'cti.ip' => 'IP Address', + 'crowdsec.id' => 'ID', + 'crowdsec.first_seen' => 'First Seen', + 'crowdsec.last_seen' => 'Last Seen', + 'crowdsec.behavior' => 'Behavior', + 'crowdsec.false_pos' => 'False Positive', + 'crowdsec.classification' => 'Classification', + 'crowdsec.score_overall' => 'Overall Score', + 'crowdsec.last_checked_at' => 'Last Checked At', + 'nerd.id' => 'ID', + 'nerd.as_name' => 'AS Name', + 'nerd.as_id' => 'AS ID', + 'nerd.ip_range' => 'IP range', + 'nerd.ip_range_rep' => 'IP range rep', + 'nerd.events' => 'Events', + 'nerd.geo_city' => 'City', + 'nerd.geo_country' => 'Country', + 'nerd.hostname' => 'Hostname', + 'nerd.first_activity' => 'First Seen', + 'nerd.last_activity' => 'Last Seen', + 'nerd.fmp' => 'FMP', + 'nerd.blacklists' => 'Blacklists', + 'nerd.rep' => 'Reputation', + 'nerd.last_checked_at' => 'Last Checked At', + ]; + } +} + +class CsvService { + private static function getFileExpirationDuration(){ + $json = file_get_contents('/var/www/html/secmon/config/cti_config.json'); + $json_data = json_decode($json, true); + return $json_data["file_validity"]; + } + + public static function getRepFromCsv($ip) + { + $expirationInHours = CsvService::getFileExpirationDuration(); + + $client = new Client(['responseConfig' => [ + 'format' => Client::FORMAT_JSON + ]]); + + $filePath = "/var/www/html/secmon/config/ip_rep.csv"; + + if (file_exists($filePath)) { + $csv = file_get_contents($filePath); + + $csvData = str_getcsv($csv, "\n", "", "#"); + + $date = []; + preg_match('/[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1]) (0[0-9]|1[0-9]|2[0-3]):([0-5][0-9])/', $csvData[0], $date); + $stamp = new \DateTime(date('Y-m-d H:i:s', strtotime($date[0]))); + $now = new \DateTime(date('Y-m-d H:i:s', strtotime('now'))); + $hoursDifference = $now->diff($stamp)->h + ($now->diff($stamp)->days * 24); + + if ($hoursDifference > $expirationInHours) { + //print("File is old enough, needs refresh\n"); + $csvData = CsvService::getCsvData($client); + $arr = CsvService::parseCsvArray($csvData); + return $arr[$ip] ?? null; + } + //print("NO NEED TO REFRESH CSV\n"); + $arr = CsvService::parseCsvArray($csvData); + return $arr[$ip] ?? null; + } else { + //print("NO FILE FOUND\n"); + $csvData = CsvService::getCsvData($client); + $arr = CsvService::parseCsvArray($csvData); + return $arr[$ip] ?? null; + } + } + + private static function parseCsvArray($csvData) + { + $arr = []; + foreach ($csvData as $row) { + $line = explode(",", $row); + $arr[$line[0]] = $line[1] ?? null; + } + return $arr; + } + + private static function getCsvData($client) + { + $response = $client->createRequest() + ->setMethod('GET') + ->setUrl('https://nerd.cesnet.cz/nerd/data/ip_rep.csv') + ->send(); + if ($response->isOk) { + //print("GOT CSV" . PHP_EOL); + $csvData = $response->getContent(); + file_put_contents('/var/www/html/secmon/config/ip_rep.csv', $csvData); + $csvData = str_getcsv($csvData, "\n", "", "#"); + return $csvData; + } + return null; + } +} \ No newline at end of file diff --git a/models/CtiNerd.php b/models/CtiNerd.php new file mode 100644 index 00000000..56464bf6 --- /dev/null +++ b/models/CtiNerd.php @@ -0,0 +1,15 @@ + null], - [['cef_severity', 'device_custom_number1', 'device_custom_number2', 'device_custom_number3', 'baseEventCount', 'device_direction', 'device_process_id', 'destination_translated_port', 'destination_process_id', 'destination_port', 'file_size', 'old_file_size', 'bytes_in', 'bytes_out', 'source_translated_port', 'source_process_id', 'source_port', 'agent_translated_zone_key', 'agent_zone_key', 'customer_key', 'destination_translated_zone_key', 'destination_zone_key', 'device_translated_zone_key', 'device_zone_key', 'source_translated_zone_key', 'source_zone_key', 'reported_duration', 'destination_ip_network_model', 'source_ip_network_model'], 'integer'], + [['cef_severity', 'device_custom_number1', 'device_custom_number2', 'device_custom_number3', 'baseEventCount', 'device_direction', 'device_process_id', 'destination_translated_port', 'destination_process_id', 'destination_port', 'file_size', 'old_file_size', 'bytes_in', 'bytes_out', 'source_translated_port', 'source_process_id', 'source_port', 'agent_translated_zone_key', 'agent_zone_key', 'customer_key', 'destination_translated_zone_key', 'destination_zone_key', 'device_translated_zone_key', 'device_zone_key', 'source_translated_zone_key', 'source_zone_key', 'reported_duration', 'destination_ip_network_model', 'source_ip_network_model', 'source_cti_id', 'destination_cti_id'], 'default', 'value' => null], + [['cef_severity', 'device_custom_number1', 'device_custom_number2', 'device_custom_number3', 'baseEventCount', 'device_direction', 'device_process_id', 'destination_translated_port', 'destination_process_id', 'destination_port', 'file_size', 'old_file_size', 'bytes_in', 'bytes_out', 'source_translated_port', 'source_process_id', 'source_port', 'agent_translated_zone_key', 'agent_zone_key', 'customer_key', 'destination_translated_zone_key', 'destination_zone_key', 'device_translated_zone_key', 'device_zone_key', 'source_translated_zone_key', 'source_zone_key', 'reported_duration', 'destination_ip_network_model', 'source_ip_network_model', 'source_cti_id', 'destination_cti_id'], 'integer'], [['cef_version', 'cef_severity', 'cef_event_class_id', 'cef_device_product', 'cef_vendor', 'cef_device_version', 'cef_name'], 'required'], [['device_custom_floating_point1', 'device_custom_floating_point2', 'device_custom_floating_point3', 'device_custom_floating_point4', 'source_geo_longitude', 'source_geo_latitude', 'destination_geo_longitude', 'destination_geo_latitude'], 'number'], [['parent_events', 'cef_extensions', 'raw_event'], 'string'], @@ -400,6 +402,8 @@ public static function columns() 'destination_geo_latitude' => [ FilterTypeEnum::COMPARE ], 'destination_ip_network_model' => [ FilterTypeEnum::COMPARE ], 'source_ip_network_model' => [ FilterTypeEnum::COMPARE ], + 'destination_cti_id' => [ FilterTypeEnum::COMPARE ], + 'source_cti_id' => [ FilterTypeEnum::COMPARE ], 'source_code' => [ FilterTypeEnum::REGEX, FilterTypeEnum::COMPARE ], 'destination_code' => [ FilterTypeEnum::REGEX, FilterTypeEnum::COMPARE ], 'parent_events' => [ FilterTypeEnum::REGEX, FilterTypeEnum::COMPARE ], @@ -581,6 +585,8 @@ public static function labels() 'destination_geo_latitude' => 'Destination Geo Latitude', 'destination_ip_network_model' => 'Destination Ip Network Model', 'source_ip_network_model' => 'Source Ip Network Model', + 'destination_cti_id' => 'Destination Ip CTI Model', + 'source_cti_id' => 'Source Ip CTI Model', 'source_code' => 'Source Code', 'destination_code' => 'Destination Code', 'parent_events' => 'Parent Events', @@ -890,6 +896,22 @@ public static function extractCefFields($cefString, $eventType) if($position != FALSE){ $event->destination_ip_network_model = $cefString[$position + strlen("dst_network_model_id=")]; } + + //map CTI model for src IP + $position = strpos($cefString, "src_cti_id="); + if($position != FALSE){ + $start_position = $position + strlen("src_cti_id="); + $end_position = strpos($cefString, " ", $start_position); + $event->source_cti_id = substr($cefString, $start_position, $end_position - $start_position); + } + + //map CTI model for dst IP + $position = strpos($cefString, "dst_cti_id="); + if($position != FALSE){ + $start_position = $position + strlen("dst_cti_id="); + $end_position = strpos($cefString, " ", $start_position); + $event->destination_cti_id = substr($cefString, $start_position, $end_position - $start_position); + } //map geoIP for src IP $position = strpos($cefString, "src_country_isoCode="); @@ -963,7 +985,6 @@ public static function extractCefFields($cefString, $eventType) $end_position = strpos($cefString, " ", $start_position); $event->destination_geo_longitude = substr($cefString, $start_position, $end_position - $start_position); } - return $event; } } diff --git a/secmon_deploy.sh b/secmon_deploy.sh index f8257a1d..0c6b3641 100755 --- a/secmon_deploy.sh +++ b/secmon_deploy.sh @@ -41,7 +41,7 @@ echo -e "${GREEN}Password successfully created${NORMAL}" echo -e "Updating passwords in configuration files" sed -i "s//$password1/g" config/db.php \ && sed -i "s//$password1/g" config/anomaly_config.ini \ -&& sed -i "s//$password1/g" config/secmon_config.ini \ +&& sed -i "s//$password1/g" config/secmon_config.yaml \ && sed -i "s//$password1/g" docker-compose.yml \ || { echo "${RED}Updating passwords in configuration file templates failed${NORMAL}" ; exit 1; } echo -e "${GREEN}Done${NORMAL}" @@ -54,6 +54,7 @@ docker build -t secmon_base -f deployment/dockerfiles/secmon_base.Dockerfile ./ && docker build -t secmon_network_model -f deployment/dockerfiles/secmon_network_model.Dockerfile ./deployment \ && docker build -t secmon_correlator -f deployment/dockerfiles/secmon_correlator.Dockerfile ./deployment \ && docker build -t secmon_db_retention -f deployment/dockerfiles/secmon_db_retention.Dockerfile ./deployment \ +&& docker build -t secmon_cti -f deployment/dockerfiles/secmon_cti.Dockerfile ./deployment \ || { echo "${RED}Building docker images failed${NORMAL}" ; exit 1; } docker compose build || { echo "${RED}Docker compose build failed${NORMAL}" ; exit 1; } diff --git a/secmon_manager.py b/secmon_manager.py index 1f870428..ecd982d0 100755 --- a/secmon_manager.py +++ b/secmon_manager.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 # encoding: utf-8 -import configparser import sys import os import time +import yaml RED = '\033[0;31m' GREEN = '\033[0;32m' @@ -12,87 +12,78 @@ NORMAL = '\033[0m' MAGENTA = '\033[0;35m' -SECMON_MAIN_CONF = './config/secmon_config.ini' +SECMON_MAIN_CONF = './config/secmon_config.yaml' SECMON_AGGREGATOR_CONF = './config/aggregator_config.ini' - def print_help(): print("Available parameters are:\n") print("\"help\" - to list all available parameters") print("\"deploy\" - to deploy SecMon") print("\"start\" - to start stopped SecMon containers") print("\"stop\" - to stop running SecMon containers") - print("\"restart\" - to restart SecMon containers") - print("\"remove\" - to remove all SecMon enrichment containers") + print("\"restart\" - to restart all SecMon containers") + print("\"remove\" - to remove all SecMon containers with database") print("\"config\" - to run initial SecMon configuration") print("\"update-rules\" - to manually update default rules\n") +#run specific enrichment module +def run_enrichment_module(module): + command = f'docker run -d {" ".join(module["args"])} --restart unless-stopped --name secmon_{module["name"].lower()} --network secmon_app-network -v ${{PWD}}:/var/www/html/secmon secmon_{module["name"].lower()}' + if os.system(command) == 0: + os.system(f'echo -e "\r\033[1A\033[0KCreating secmon_{module["name"].lower()} ... {GREEN}done{NORMAL}"') + else: + os.system(f'echo -e "\r\033[1A\033[0KCreating secmon_{module["name"].lower()} ... {RED}failed{NORMAL}"') -# Return all enabled enrichment modules in "secmon_config.ini" -def get_enabled_enr_modules(): - config = configparser.ConfigParser() - config.read(SECMON_MAIN_CONF) - - enabled_enrichment_modules = [] - for module in all_enrichment_modules: - if config.get('ENRICHMENT', module) == 'true': - enabled_enrichment_modules.append(module) - - return enabled_enrichment_modules - +def run_correlator_module(): + command = f'docker run -d --restart unless-stopped --name secmon_correlator --network secmon_app-network -v ${{PWD}}:/var/www/html/secmon secmon_correlator' + if os.system(command) == 0: + os.system(f'echo -e "\r\033[1A\033[0KCreating secmon_correlator ... {GREEN}done{NORMAL}"') + else: + os.system(f'echo -e "\r\033[1A\033[0KCreating secmon_correlator ... {RED}failed{NORMAL}"') -# Run specific enrichment module -def create_enrichment_modules(): - print(YELLOW, '\nCreating secmon enrichment modules:', NORMAL) +# Method for starting stopped containers +def start_secmon_containers(enabled_enrichment_modules): + print(YELLOW,'\nStarting secmon modules:', NORMAL) + os.system('docker compose -p secmon start') - enabled_enrichment_modules = get_enabled_enr_modules() for module in enabled_enrichment_modules: - command = f'docker run -d --restart unless-stopped --name secmon_{module} --network secmon_app-network \ - -v ${{PWD}}:/var/www/html/secmon secmon_{module}' - if os.system(command) == 0: - os.system(f'echo -e "\r\033[1A\033[0KCreating secmon_{module} ... {GREEN}done{NORMAL}"') - else: - os.system(f'echo -e "\r\033[1A\033[0KCreating secmon_{module} ... {RED}failed{NORMAL}"') + if os.system(f'docker container inspect secmon_{module["name"].lower()} > /dev/null 2>&1') == 0: + if os.system(f'docker start secmon_{module["name"].lower()}') == 0: + os.system(f'echo -e "\r\033[1A\033[0KStarting secmon_{module["name"].lower()} ... {GREEN}done{NORMAL}"') + else: + os.system(f'echo -e "\r\033[1A\033[0KStarting secmon_{module["name"].lower()} ... {RED}failed{NORMAL}"') +#method for restarting running/stopped containers +def restart_secmon_containers(all_modules, enabled_enrichment_modules): + stop_secmon_containers(all_modules) + remove_secmon_containers(all_modules) -# Method for starting stopped containers -def start_secmon_containers(): - print(YELLOW, '\nStarting secmon modules:', NORMAL) + os.system('docker compose -p secmon restart') - enabled_enrichment_modules = get_enabled_enr_modules() + print(YELLOW,'\nCreating SecMon enrichment modules:', NORMAL) for module in enabled_enrichment_modules: - if os.system(f'docker container inspect secmon_{module} > /dev/null 2>&1') == 0: - if os.system(f'docker start secmon_{module}') == 0: - os.system(f'echo -e "\r\033[1A\033[0KStarting secmon_{module} ... {GREEN}done{NORMAL}"') - else: - os.system(f'echo -e "\r\033[1A\033[0KStarting secmon_{module} ... {RED}failed{NORMAL}"') - os.system('docker compose start') - + run_enrichment_module(module) # Method for stopping running containers -def stop_secmon_containers(): +def stop_secmon_containers(all_modules): print(YELLOW, '\nStopping secmon modules:', NORMAL) - - for module in all_enrichment_modules: - if os.system(f'docker container inspect secmon_{module} > /dev/null 2>&1') == 0: - if os.system(f'docker stop secmon_{module}') == 0: - os.system(f'echo -e "\r\033[1A\033[0KStopping secmon_{module} ... {GREEN}done{NORMAL}"') + for module in all_modules: + if os.system(f'docker container inspect secmon_{module["name"].lower()} > /dev/null 2>&1') == 0: + if os.system(f'docker stop secmon_{module["name"].lower()}') == 0: + os.system(f'echo -e "\r\033[1A\033[0KStopping secmon_{module["name"].lower()} ... {GREEN}done{NORMAL}"') else: - os.system(f'echo -e "\r\033[1A\033[0KStopping secmon_{module} ... {RED}failed{NORMAL}"') - os.system('docker compose stop') - + os.system(f'echo -e "\r\033[1A\033[0KStopping secmon_{module["name"].lower()} ... {RED}failed{NORMAL}"') + os.system('docker compose -p secmon stop') # Method for removing stopped containers -def remove_secmon_containers(): +def remove_secmon_containers(all_modules): print(YELLOW, '\nRemoving secmon modules:', NORMAL) - - for module in all_enrichment_modules: - if os.system(f'docker container inspect secmon_{module} > /dev/null 2>&1') == 0: - if os.system(f'docker rm secmon_{module}') == 0: - os.system(f'echo -e "\r\033[1A\033[0KRemoving secmon_{module} ... {GREEN}done{NORMAL}"') + for module in all_modules: + if os.system(f'docker container inspect secmon_{module["name"].lower()} > /dev/null 2>&1') == 0: + if os.system(f'docker rm secmon_{module["name"].lower()}') == 0: + os.system(f'echo -e "\r\033[1A\033[0KRemoving secmon_{module["name"].lower()} ... {GREEN}done{NORMAL}"') else: - os.system(f'echo -e "\r\033[1A\033[0KRemoving secmon_{module} ... {RED}failed{NORMAL}"') - + os.system(f'echo -e "\r\033[1A\033[0KRemoving secmon_{module["name"].lower()} ... {RED}failed{NORMAL}"') def path_validation(path, input_data): return path in input_data @@ -110,22 +101,18 @@ def log_input_device_name_validation(name, input_data, index): # Create temp config for deployment -def create_temp_config(): - # Read configuration file - config = configparser.ConfigParser() - config.read(SECMON_MAIN_CONF) - - # Validate secmon_config.ini - if not validate(config): +def create_temp_config(secmon_config): + # Validate secmon_config.yaml + if not validate(secmon_config): sys.exit() # Write data to temp config for system services port = 9000 aggregator_conf_file = open(SECMON_AGGREGATOR_CONF, "w+") + aggregator_conf_file.write("Log_input: %s\nName: %s\n" % (secmon_config['DEVICE']['log_input'], secmon_config['DEVICE']['name'])) - aggregator_conf_file.write(f"Log_input: {config.get('DEVICE', 'log_input')}\nName: {config.get('DEVICE', 'name')}\n") - aggregator_conf_file.write(f"Nor_input_NP: {config.get('NORMALIZATION', 'input_NP')}\nNor_output_NP: {config.get('NORMALIZATION', 'output_NP')}\n") - aggregator_conf_file.write(f"Cor_input_NP: {config.get('CORRELATION', 'input_NP')}\nCor_output_NP: {config.get('CORRELATION', 'output_NP')}\n") + aggregator_conf_file.write("Nor_input_NP: %s\nNor_output_NP: %s\n" % (secmon_config['NORMALIZATION']['input_NP'], secmon_config['NORMALIZATION']['output_NP'])) + aggregator_conf_file.write("Cor_input_NP: %s\nCor_output_NP: %s\n" % (secmon_config['CORRELATION']['input_NP'], secmon_config['CORRELATION']['output_NP'])) # Write 0MQ port for aggregator aggregator_conf_file.write("Aggregator: %d\n" % port) @@ -133,19 +120,11 @@ def create_temp_config(): # Write 0MQ port for normalizer aggregator_conf_file.write("Normalizer: %d\n" % port) - # Write 0MQ port for geoip and - if config.get('ENRICHMENT', 'geoip').lower() == "true": - aggregator_conf_file.write("Geoip: %d\n" % port) - - # Write 0MQ port for network_model - if config.get('ENRICHMENT', 'network_model').lower() == "true": - aggregator_conf_file.write("Network_model: %d\n" % port) - - # !!! ADD HERE ANY NEW ENRICHMENT MODULE 0MQ port CONFIG !!! - - # if config.get('ENRICHMENT', 'rep_ip').lower() == "true": - # #write 0MQ port for rep_ip - # aggregator_conf_file.write("Rep_ip: %d\n" % port) + # Write 0MQ ports for every enrichment module in the config + for module in secmon_config["ENRICHMENT"]: + if module["enabled"] and module["name"] != "correlator": + aggregator_conf_file.write(f'{module["name"]}: {port}\n') + print(module["name"]) aggregator_conf_file.close() @@ -156,60 +135,60 @@ def validate(config): error = 0 # log_input path validation - if not path_validation("/var/log/", config.get('DEVICE', 'log_input')): + if not path_validation("/var/log/", config['DEVICE']['log_input']): error_msg += '\n' + "Log input must contain /var/log/ path! Please change the path." error = 1 # device name in log_input path validation - if not log_input_device_name_validation(config.get('DEVICE', 'name'), config.get('DEVICE', 'log_input'), 3): + if not log_input_device_name_validation(config['DEVICE']['name'], config['DEVICE']['log_input'], 3): error_msg += '\n' + ("Source directory of log input path must have a same name as device name! Please rename \ source directory on log input path.") error = 1 # normalization input named pipe path validation - if not path_validation("/var/log/", config.get('NORMALIZATION', 'input_NP')): + if not path_validation("/var/log/", config['NORMALIZATION']['input_NP']): error_msg += '\n' + ("Path of normalization INPUT naped pipe must contain /var/log/ path! Please change the \ path.") error = 1 # device name in normalization input named pipe validation - if not log_input_device_name_validation(config.get('DEVICE', 'name'), config.get('NORMALIZATION', 'input_NP'), 3): + if not log_input_device_name_validation(config['DEVICE']['name'], config['NORMALIZATION']['input_NP'], 3): error_msg += '\n' + ("Source directory of normalization INPUT named pipe must have a same name as device name! \ Please rename source directory on log input path.") error = 1 # normalization output named pipe path validation - if not path_validation("/var/log/", config.get('NORMALIZATION', 'output_NP')): + if not path_validation("/var/log/", config['NORMALIZATION']['output_NP']): error_msg += '\n' + ("Path of normalization OUTPUT naped pipe must contain /var/log/ path! Please change the \ path.") error = 1 # device name in normalization output named pipe validation - if not log_input_device_name_validation(config.get('DEVICE', 'name'), config.get('NORMALIZATION', 'output_NP'), 3): + if not log_input_device_name_validation(config['DEVICE']['name'], config['NORMALIZATION']['output_NP'], 3): error_msg += '\n' + ("Source directory of normalization OUTPUT named pipe must have a same name as device name! \ Please rename source directory on log input path.") error = 1 # correlation input named pipe path validation - if not path_validation("/var/www/html/", config.get('CORRELATION', 'input_NP')): + if not path_validation("/var/www/html/", config['CORRELATION']['input_NP']): error_msg += '\n' + ("Path of correlation INPUT named pipe must contain /var/www/html/ path! Please change the \ path.") error = 1 # device name in correlation input named pipe validation - if not log_input_device_name_validation(config.get('DEVICE', 'name'), config.get('CORRELATION', 'input_NP'), 4): + if not log_input_device_name_validation(config['DEVICE']['name'], config['CORRELATION']['input_NP'], 4): error_msg += '\n' + ("Source directory of correlation INPUT named pipe must have a same name as device name! \ Please rename source directory on log input path.") error = 1 # correlation output named pipe path validation - if not path_validation("/var/www/html/", config.get('CORRELATION', 'output_NP')): + if not path_validation("/var/www/html/", config['CORRELATION']['output_NP']): error_msg += '\n' + ("Path of correlation OUTPUT named pipe must contain /var/www/html/ path! Please change \ the path.") error = 1 # device name in correlation output named pipe validation - if not log_input_device_name_validation(config.get('DEVICE', 'name'), config.get('CORRELATION', 'output_NP'), 4): + if not log_input_device_name_validation(config['DEVICE']['name'], config['CORRELATION']['output_NP'], 4): error_msg += '\n' + ("Source directory of correlation OUTPUT named pipe must have a same name as device name! \ Please rename source directory on log input path.") error = 1 @@ -220,40 +199,52 @@ def validate(config): else: return True +ALL_MODULES = [] +ENABLED_ENRICHMENT_MODULES = [] + +def get_config(): + global ENABLED_ENRICHMENT_MODULES + global ALL_MODULES + with open(SECMON_MAIN_CONF) as secmon_config_file: + secmon_config = yaml.safe_load(secmon_config_file) -# Script main entry point + for module in secmon_config["ENRICHMENT"]: + if module["enabled"]: + ENABLED_ENRICHMENT_MODULES.append(module) + print(module["name"]) + ALL_MODULES.append(module) + return secmon_config if len(sys.argv) < 2 or sys.argv[1] == "help": print_help() sys.exit() -all_enrichment_modules = ['geoip', 'network_model', 'correlator'] - # Start stopped containers if sys.argv[1] == "start": - start_secmon_containers() + get_config() + start_secmon_containers(ENABLED_ENRICHMENT_MODULES) sys.exit() # Stop running containers if sys.argv[1] == "stop": - stop_secmon_containers() + get_config() + stop_secmon_containers(ALL_MODULES) sys.exit() # Stop and remove all secmon containers if sys.argv[1] == "remove": - stop_secmon_containers() - remove_secmon_containers() - os.system('docker compose down') + get_config() + stop_secmon_containers(ALL_MODULES) + remove_secmon_containers(ALL_MODULES) + os.system('docker compose -p secmon down') sys.exit() # Restart running containers if sys.argv[1] == "restart": + secmon_config = get_config() print(YELLOW, '\nRestarting SecMon modules:', NORMAL) - create_temp_config() - stop_secmon_containers() - remove_secmon_containers() - create_enrichment_modules() - os.system('docker compose restart') + create_temp_config(secmon_config) + restart_secmon_containers(ALL_MODULES, ENABLED_ENRICHMENT_MODULES) sys.exit() # Manually run secmon configuration @@ -282,7 +273,8 @@ def validate(config): else: print(YELLOW, "Initial configuration already executed. Skipping step.", NORMAL) - create_temp_config() + secmon_config = get_config() + create_temp_config(secmon_config) answer = input("Deploying SecMon will remove all existing SecMon containers and existing SecMon database.\n" "This process also includes setting up different config files and creating new SecMon containers.\n" @@ -291,9 +283,9 @@ def validate(config): sys.exit() elif answer.lower() == "y": # Stop and remove enrichment modules - stop_secmon_containers() - remove_secmon_containers() - os.system('docker compose down') + stop_secmon_containers(ALL_MODULES) + remove_secmon_containers(ALL_MODULES) + os.system('docker compose -p secmon down') # Auto execute 'secmon_deploy.sh' if os.system('sudo bash ./secmon_deploy.sh') != 0: # set sudo @@ -301,7 +293,9 @@ def validate(config): sys.exit() os.system('docker compose -p secmon up -d') - create_enrichment_modules() + + for module in ENABLED_ENRICHMENT_MODULES: + run_enrichment_module(module) # Check the status of the database and wait until it is ready to receive connections os.system( 'docker logs secmon_db 2>&1 | grep -q "listening on IPv4 address \\"0.0.0.0\\", port 5432" && echo \ @@ -317,6 +311,7 @@ def validate(config): os.system(f'echo -n "Initializing SecMon admin user ... {GREEN}"') os.system('curl 127.0.0.1:8080/secmon/web/user/init') + restart_secmon_containers(ALL_MODULES, ENABLED_ENRICHMENT_MODULES) print(MAGENTA, "\nDeployment successful. SecMon is now live.", NORMAL) sys.exit() else: diff --git a/secmon_preconfig.sh b/secmon_preconfig.sh index e77de4ce..b96a2c45 100755 --- a/secmon_preconfig.sh +++ b/secmon_preconfig.sh @@ -95,12 +95,13 @@ echo -e "${GREEN}Done${NORMAL}" echo -e "Copying configuration file templates" cp deployment/config_templates/db.php config/ \ && cp deployment/config_templates/anomaly_config.ini config/ \ -&& cp deployment/config_templates/secmon_config.ini config/ \ +&& cp deployment/config_templates/secmon_config.yaml config/ \ +&& cp deployment/config_templates/cti_config.json config/ \ && cp deployment/config_templates/docker-compose.yml . \ || { echo -e "${RED}Copying configuration file templates failed!${NORMAL}" ; exit 1; } echo -e "${GREEN}Done${NORMAL}" -# Download rules from repository configured in secmon_config.ini +# Download rules from repository configured in secmon_config.yaml echo -e "${YELLOW}Starting download of SecMon rules${NORMAL}" python3 ./commands/rules_downloader.py os \ || { echo -e "${RED}Download of SecMon rules failed${NORMAL}" ; exit 1; } diff --git a/views/security-events/view.php b/views/security-events/view.php index 93c31bb3..5006cdf6 100644 --- a/views/security-events/view.php +++ b/views/security-events/view.php @@ -3,6 +3,7 @@ use macgyer\yii2materializecss\widgets\data\DetailView; use yii\helpers\Html; use app\models\NetworkModel; +use app\models\CtiModel; /* @var $this yii\web\View */ /* @var $model app\models\SecurityEvents */ @@ -10,6 +11,8 @@ $this->params['title'] = 'Security Event ID: ' . $model->id; $this->params['src_device'] = NetworkModel::getNetworkDevice($model->source_ip_network_model); $this->params['dst_device'] = NetworkModel::getNetworkDevice($model->destination_ip_network_model); +$this->params['src_cti_model'] = CtiModel::getCtiInfo($model->source_cti_id); +$this->params['dst_cti_model'] = CtiModel::getCtiInfo($model->destination_cti_id); ?>
@@ -90,6 +93,263 @@ ]) ?>
+
    +
  • +
    securityCTI information: params["src_cti_model"]['crowdsec']->classification ?? null ?>
    +
    $this->params["src_cti_model"], + 'attributes' => [ + [ + "label" => 'IP', + 'value' => $this->params["src_cti_model"]['ip'] + ], + [ + "label" => 'FMP', + 'value' => $this->params["src_cti_model"]['nerd']->fmp ?? null + ], + [ + "label" => 'Reputational score', + 'format' => 'raw', + 'value' => function () { + $tag = null; + $reputationNerd = null; + $scoreCrowd = null; + if ($this->params["src_cti_model"]['reputation'] !== null) { + $reputationNerd = $this->params["src_cti_model"]['reputation']; + $iconNameNerd = 'dangerous'; + $colorClassNerd = "danger"; + + if ($reputationNerd < 0.50) { + $iconNameNerd = 'check_circle'; + $colorClassNerd = "safe"; + } elseif ($reputationNerd < 0.75) { + $iconNameNerd = 'warning'; + $colorClassNerd = "warning"; + } + } + if ($this->params["src_cti_model"]['crowdsec'] !== null) { + $scoreCrowd = $this->params["src_cti_model"]['crowdsec']->score_overall; + $iconNameCrowd = 'dangerous'; + $colorClassCrowd = "danger"; + + if ($scoreCrowd < 1) { + $iconNameCrowd = 'check_circle'; + $colorClassCrowd = "safe"; + } elseif ($scoreCrowd < 4) { + $iconNameCrowd = 'warning'; + $colorClassCrowd = "warning"; + } + } + $reputationNerdDiv = null; + if ($reputationNerd !== null) { + $reputationNerdDiv = Html::tag('i', $iconNameNerd, ['class' => "material-icons vertical-top " . $colorClassNerd]) ." ". Html::tag('div', $reputationNerd, ['class' => "vertical-top"]) . " " . Html::tag('p', '[NERD]', ['class' => "vertical-top gray"]) . "
    "; + } + $scoreCrowdDiv = null; + if ($scoreCrowd !== null){ + $scoreCrowdDiv = Html::tag('i', $iconNameCrowd, ['class' => "material-icons vertical-top " . $colorClassCrowd]) ." ". Html::tag('div', $scoreCrowd, ['class' => "vertical-top"]) . " " . Html::tag('p', '[Crowdsec]', ['class' => "vertical-top gray"]); + } + $tag = $reputationNerdDiv . $scoreCrowdDiv; + if ($tag) { + return Html::tag('div', $tag); + } + }, + ], + [ + "label" => 'Blacklists', + 'value' => $this->params["src_cti_model"]['nerd']->blacklists ?? null + ], + [ + "label" => 'Classification', + 'value' => $this->params["src_cti_model"]['crowdsec']->classification ?? null + ], + [ + "label" => "events", + 'format' => 'raw', + 'value' => function(){ + $tag = null; + if($this->params["src_cti_model"]["events"]?->nerd != null){ + $tag = $tag . Html::tag('div', $this->params["src_cti_model"]["events"]->nerd, ['class' => "vertical-top"]) ." ". Html::tag('p', '[NERD]', ['class' => "vertical-top gray"]) . "
    "; + } + if($this->params["src_cti_model"]["events"]?->crowd != null){ + $tag = $tag . Html::tag('div', $this->params["src_cti_model"]["events"]->crowd, ['class' => "vertical-top"]) ." ". Html::tag('p', '[Crowdsec]', ['class' => "vertical-top gray"]); + } + if ($tag){ + return Html::tag('div', $tag); + } + return null; + } + ], + [ + "label" => "Geolocation", + 'format' => 'raw', + 'value' => function(){ + $tag = null; + $crowd = null; + $nerd = null; + if($this->params["src_cti_model"]["city"]?->nerd != null){ + $nerd = $nerd . Html::tag('div', $this->params["src_cti_model"]["city"]->nerd . ", ", ['class' => "vertical-top"]); + } + if($this->params["src_cti_model"]["country"]?->nerd != null){ + $nerd = $nerd . Html::tag('div', $this->params["src_cti_model"]["country"]->nerd, ['class' => "vertical-top"]); + } + if ($nerd){ + $nerd = $nerd ." ". Html::tag('p', '[NERD]', ['class' => "vertical-top gray"]) . "
    "; + } + + if($this->params["src_cti_model"]["city"]?->crowd != null){ + $crowd = $crowd . Html::tag('div', $this->params["src_cti_model"]["city"]->crowd . ", ", ['class' => "vertical-top"]); + } + if($this->params["src_cti_model"]["country"]?->crowd != null){ + $crowd = $crowd . Html::tag('div', $this->params["src_cti_model"]["country"]->crowd, ['class' => "vertical-top"]); + } + if ($crowd){ + $crowd = $crowd ." ". Html::tag('p', '[Crowdsec]', ['class' => "vertical-top gray"]); + } + + $tag = $nerd . $crowd; + if ($tag){ + return Html::tag('div', $tag); + } + } + ], + [ + "label" => "IP range", + 'format' => 'raw', + 'value' => function(){ + $tag = null; + if ($this->params["src_cti_model"]["ip_range"]?->nerd != null && ($this->params["src_cti_model"]["ip_range"]->nerd == $this->params["src_cti_model"]["ip_range"]->crowd)){ + return Html::tag('div', $this->params["src_cti_model"]["ip_range"]->nerd, ['class' => "vertical-top"]) ." ". Html::tag('p', '[NERD, Crowdsec]', ['class' => "vertical-top gray"]); + } + if($this->params["src_cti_model"]["ip_range"]?->nerd != null){ + $tag = $tag . Html::tag('div', $this->params["src_cti_model"]["ip_range"]->nerd, ['class' => "vertical-top"]) ." ". Html::tag('p', '[NERD]', ['class' => "vertical-top gray"]) . "
    "; + } + if($this->params["src_cti_model"]["ip_range"]?->crowd != null){ + $tag = $tag . Html::tag('div', $this->params["src_cti_model"]["ip_range"]->crowd, ['class' => "vertical-top"]) ." ". Html::tag('p', '[Crowdsec]', ['class' => "vertical-top gray"]); + } + if ($tag){ + return Html::tag('div', $tag); + } + } + ], + [ + "label" => "ip_range rep", + 'format' => 'raw', + 'value' => function(){ + $tag = null; + if($this->params["src_cti_model"]["ip_range_rep"]?->nerd != null){ + $tag = $tag . Html::tag('div', $this->params["src_cti_model"]["ip_range_rep"]->nerd, ['class' => "vertical-top"]) ." ". Html::tag('p', '[NERD]', ['class' => "vertical-top gray"]) . "
    "; + } + if($this->params["src_cti_model"]["ip_range_rep"]?->crowd != null){ + $tag = $tag . Html::tag('div', $this->params["src_cti_model"]["ip_range_rep"]->crowd, ['class' => "vertical-top"]) ." ". Html::tag('p', '[Crowdsec]', ['class' => "vertical-top gray"]); + } + if ($tag){ + return Html::tag('div', $tag); + } + } + ], + [ + "label" => "Hostname", + 'format' => 'raw', + 'value' => function(){ + $tag = null; + if ($this->params["src_cti_model"]["hostname"]?->nerd != null && ($this->params["src_cti_model"]["hostname"]->nerd == $this->params["src_cti_model"]["hostname"]->crowd)){ + return Html::tag('div', $this->params["src_cti_model"]["hostname"]->nerd, ['class' => "vertical-top"]) ." ". Html::tag('p', '[NERD, Crowdsec]', ['class' => "vertical-top gray"]); + } + + if($this->params["src_cti_model"]["hostname"]?->nerd != null){ + $tag = $tag . Html::tag('div', $this->params["src_cti_model"]["hostname"]->nerd, ['class' => "vertical-top"]) ." ". Html::tag('p', '[NERD]', ['class' => "vertical-top gray"]) . "
    "; + } + if($this->params["src_cti_model"]["hostname"]?->crowd != null){ + $tag = $tag . Html::tag('div', $this->params["src_cti_model"]["hostname"]->crowd, ['class' => "vertical-top"]) ." ". Html::tag('p', '[Crowdsec]', ['class' => "vertical-top gray"]); + } + if ($tag){ + return Html::tag('div', $tag); + } + } + ], + [ + "label" => "AS name (ID)", + 'format' => 'raw', + 'value' => function(){ + $tag = null; + $nerd = null; + $crowd = null; + if($this->params["src_cti_model"]["as_name"]?->nerd != null){ + $nerd = $nerd . Html::tag('div', $this->params["src_cti_model"]["as_name"]->nerd . " ", ['class' => "vertical-top"]); + } + if($this->params["src_cti_model"]["as_num"]?->nerd != null){ + $nerd = $nerd . Html::tag('div', "(".$this->params["src_cti_model"]["as_num"]->nerd.")", ['class' => "vertical-top"]); + } + if ($nerd){ + $nerd = $nerd ." ". Html::tag('p', '[NERD]', ['class' => "vertical-top gray"]) . "
    "; + } + if($this->params["src_cti_model"]["as_name"]?->crowd != null){ + $crowd = $crowd . Html::tag('div', $this->params["src_cti_model"]["as_name"]->crowd . " ", ['class' => "vertical-top"]); + } + if($this->params["src_cti_model"]["as_num"]?->crowd != null){ + $crowd = $crowd . Html::tag('div', "(".$this->params["src_cti_model"]["as_num"]->crowd.")", ['class' => "vertical-top"]); + } + if ($crowd){ + $crowd = $crowd . " ". Html::tag('p', '[Crowdsec]', ['class' => "vertical-top gray"]); + } + $tag = $nerd . $crowd; + if ($tag){ + return Html::tag('div', $tag); + } + } + ], + [ + "label" => "First seen", + 'format' => 'raw', + 'value' => function(){ + $tag = null; + if($this->params["src_cti_model"]["first_seen"]?->nerd != null){ + $tag = $tag . Html::tag('div', $this->params["src_cti_model"]["first_seen"]->nerd, ['class' => "vertical-top"]) ." ". Html::tag('p', '[NERD]', ['class' => "vertical-top gray"]) . "
    "; + } + if($this->params["src_cti_model"]["first_seen"]?->crowd != null){ + $tag = $tag . Html::tag('div', $this->params["src_cti_model"]["first_seen"]->crowd, ['class' => "vertical-top"]) ." ". Html::tag('p', '[Crowdsec]', ['class' => "vertical-top gray"]); + } + if ($tag){ + return Html::tag('div', $tag); + } + } + ], + [ + "label" => "Last seen", + 'format' => 'raw', + 'value' => function(){ + $tag = null; + if($this->params["src_cti_model"]["last_seen"]?->nerd != null){ + $tag = $tag . Html::tag('div', $this->params["src_cti_model"]["last_seen"]->nerd, ['class' => "vertical-top"]) ." ". Html::tag('p', '[NERD]', ['class' => "vertical-top gray"]) . "
    "; + } + if($this->params["src_cti_model"]["last_seen"]?->crowd != null){ + $tag = $tag . Html::tag('div', $this->params["src_cti_model"]["last_seen"]->crowd, ['class' => "vertical-top"]) ." ". Html::tag('p', '[Crowdsec]', ['class' => "vertical-top gray"]); + } + if ($tag){ + return Html::tag('div', $tag); + } + } + ], + [ + "label" => 'Last API update', + 'format' => 'raw', + 'value' => function(){ + $tag = null; + if($this->params["src_cti_model"]['nerd'] != null){ + $tag = $tag . Html::tag('div', $this->params["src_cti_model"]['nerd']->last_checked_at, ['class' => "vertical-top"]) ." ". Html::tag('p', '[NERD]', ['class' => "vertical-top gray"]) . "
    "; + } + if($this->params["src_cti_model"]['crowdsec'] != null){ + $tag = $tag . Html::tag('div', $this->params["src_cti_model"]['crowdsec']->last_checked_at, ['class' => "vertical-top"]) ." ". Html::tag('p', '[Crowdsec]', ['class' => "vertical-top gray"]); + } + if ($tag){ + return Html::tag('div', $tag); + } + } + ], + ], + ]) ?>
    +
    • +
  • laptopNetwork model information: params['src_device']->description?>
    @@ -153,6 +413,268 @@ ]) ?>
+
    +
  • +
    securityCTI information: params["dst_cti_model"]['crowdsec']->classification ?? null ?>
    +
    $this->params["dst_cti_model"], + 'attributes' => [ + [ + "label" => 'IP', + 'value' => $this->params["dst_cti_model"]['ip'] + ], + [ + "label" => 'Reputation', + 'value' => $this->params["dst_cti_model"]['reputation'] + ], + [ + "label" => 'FMP', + 'value' => $this->params["dst_cti_model"]['nerd']->fmp ?? null + ], + [ + "label" => 'Reputational score', + 'format' => 'raw', + 'value' => function () { + $tag = null; + $reputationNerd = null; + $scoreCrowd = null; + + if ($this->params["dst_cti_model"]['reputation'] !== null) { + $reputationNerd = $this->params["dst_cti_model"]['reputation']; + $iconNameNerd = 'dangerous'; + $colorClassNerd = "danger"; + + if ($reputationNerd < 0.50) { + $iconNameNerd = 'check_circle'; + $colorClassNerd = "safe"; + } elseif ($reputationNerd < 0.75) { + $iconNameNerd = 'warning'; + $colorClassNerd = "warning"; + } + } + if ($this->params["dst_cti_model"]['crowdsec'] !== null) { + $scoreCrowd = $this->params["dst_cti_model"]['crowdsec']->score_overall; + $iconNameCrowd = 'dangerous'; + $colorClassCrowd = "danger"; + + if ($scoreCrowd < 1) { + $iconNameCrowd = 'check_circle'; + $colorClassCrowd = "safe"; + } elseif ($scoreCrowd < 4) { + $iconNameCrowd = 'warning'; + $colorClassCrowd = "warning"; + } + } + $reputationNerdDiv = null; + if ($reputationNerd !== null) { + $reputationNerdDiv = Html::tag('i', $iconNameNerd, ['class' => "material-icons vertical-top " . $colorClassNerd]) ." ". Html::tag('div', $reputationNerd, ['class' => "vertical-top"]) . " " . Html::tag('p', '[NERD]', ['class' => "vertical-top gray"]) . "
    "; + } + $scoreCrowdDiv = null; + if ($scoreCrowd !== null){ + $scoreCrowdDiv = Html::tag('i', $iconNameCrowd, ['class' => "material-icons vertical-top " . $colorClassCrowd]) ." ". Html::tag('div', $scoreCrowd, ['class' => "vertical-top"]) . " " . Html::tag('p', '[Crowdsec]', ['class' => "vertical-top gray"]); + } + $tag = $reputationNerdDiv . $scoreCrowdDiv; + if ($tag) { + return Html::tag('div', $tag); + } + }, + ], + [ + "label" => 'Blacklists', + 'value' => $this->params["dst_cti_model"]['nerd']->blacklists ?? null + ], + [ + "label" => 'Classification', + 'value' => $this->params["dst_cti_model"]['crowdsec']->classification ?? null + ], + [ + "label" => "events", + 'format' => 'raw', + 'value' => function(){ + $tag = null; + if($this->params["dst_cti_model"]["events"]?->nerd != null){ + $tag = $tag . Html::tag('div', $this->params["dst_cti_model"]["events"]->nerd, ['class' => "vertical-top"]) ." ". Html::tag('p', '[NERD]', ['class' => "vertical-top gray"]) . "
    "; + } + if($this->params["dst_cti_model"]["events"]?->crowd != null){ + $tag = $tag . Html::tag('div', $this->params["dst_cti_model"]["events"]->crowd, ['class' => "vertical-top"]) ." ". Html::tag('p', '[Crowdsec]', ['class' => "vertical-top gray"]); + } + if ($tag){ + return Html::tag('div', $tag); + } + return null; + } + ], + [ + "label" => "Geolocation", + 'format' => 'raw', + 'value' => function(){ + $tag = null; + $crowd = null; + $nerd = null; + if($this->params["dst_cti_model"]["city"]?->nerd != null){ + $nerd = $nerd . Html::tag('div', $this->params["dst_cti_model"]["city"]->nerd . ", ", ['class' => "vertical-top"]); + } + if($this->params["dst_cti_model"]["country"]?->nerd != null){ + $nerd = $nerd . Html::tag('div', $this->params["dst_cti_model"]["country"]->nerd, ['class' => "vertical-top"]); + } + if ($nerd){ + $nerd = $nerd ." ". Html::tag('p', '[NERD]', ['class' => "vertical-top gray"]) . "
    "; + } + + if($this->params["dst_cti_model"]["city"]?->crowd != null){ + $crowd = $crowd . Html::tag('div', $this->params["dst_cti_model"]["city"]->crowd . ", ", ['class' => "vertical-top"]); + } + if($this->params["dst_cti_model"]["country"]?->crowd != null){ + $crowd = $crowd . Html::tag('div', $this->params["dst_cti_model"]["country"]->crowd, ['class' => "vertical-top"]); + } + if ($crowd){ + $crowd = $crowd ." ". Html::tag('p', '[Crowdsec]', ['class' => "vertical-top gray"]); + } + + $tag = $nerd . $crowd; + if ($tag){ + return Html::tag('div', $tag); + } + } + ], + [ + "label" => "IP range", + 'format' => 'raw', + 'value' => function(){ + $tag = null; + if ($this->params["dst_cti_model"]["ip_range"]?->nerd != null && ($this->params["dst_cti_model"]["ip_range"]->nerd == $this->params["dst_cti_model"]["ip_range"]->crowd)){ + return Html::tag('div', $this->params["dst_cti_model"]["ip_range"]->nerd, ['class' => "vertical-top"]) ." ". Html::tag('p', '[NERD, Crowdsec]', ['class' => "vertical-top gray"]); + } + if($this->params["dst_cti_model"]["ip_range"]?->nerd != null){ + $tag = $tag . Html::tag('div', $this->params["dst_cti_model"]["ip_range"]->nerd, ['class' => "vertical-top"]) ." ". Html::tag('p', '[NERD]', ['class' => "vertical-top gray"]) . "
    "; + } + if($this->params["dst_cti_model"]["ip_range"]?->crowd != null){ + $tag = $tag . Html::tag('div', $this->params["dst_cti_model"]["ip_range"]->crowd, ['class' => "vertical-top"]) ." ". Html::tag('p', '[Crowdsec]', ['class' => "vertical-top gray"]); + } + if ($tag){ + return Html::tag('div', $tag); + } + } + ], + [ + "label" => "ip_range rep", + 'format' => 'raw', + 'value' => function(){ + $tag = null; + if($this->params["dst_cti_model"]["ip_range_rep"]?->nerd != null){ + $tag = $tag . Html::tag('div', $this->params["dst_cti_model"]["ip_range_rep"]->nerd, ['class' => "vertical-top"]) ." ". Html::tag('p', '[NERD]', ['class' => "vertical-top gray"]) . "
    "; + } + if($this->params["dst_cti_model"]["ip_range_rep"]?->crowd != null){ + $tag = $tag . Html::tag('div', $this->params["dst_cti_model"]["ip_range_rep"]->crowd, ['class' => "vertical-top"]) ." ". Html::tag('p', '[Crowdsec]', ['class' => "vertical-top gray"]); + } + if ($tag){ + return Html::tag('div', $tag); + } + } + ], + [ + "label" => "Hostname", + 'format' => 'raw', + 'value' => function(){ + $tag = null; + if ($this->params["dst_cti_model"]["hostname"]?->nerd != null && ($this->params["dst_cti_model"]["hostname"]->nerd == $this->params["dst_cti_model"]["hostname"]->crowd)){ + return Html::tag('div', $this->params["dst_cti_model"]["hostname"]->nerd, ['class' => "vertical-top"]) ." ". Html::tag('p', '[NERD, Crowdsec]', ['class' => "vertical-top gray"]); + } + + if($this->params["dst_cti_model"]["hostname"]?->nerd != null){ + $tag = $tag . Html::tag('div', $this->params["dst_cti_model"]["hostname"]->nerd, ['class' => "vertical-top"]) ." ". Html::tag('p', '[NERD]', ['class' => "vertical-top gray"]) . "
    "; + } + if($this->params["dst_cti_model"]["hostname"]?->crowd != null){ + $tag = $tag . Html::tag('div', $this->params["dst_cti_model"]["hostname"]->crowd, ['class' => "vertical-top"]) ." ". Html::tag('p', '[Crowdsec]', ['class' => "vertical-top gray"]); + } + if ($tag){ + return Html::tag('div', $tag); + } + } + ], + [ + "label" => "AS name (ID)", + 'format' => 'raw', + 'value' => function(){ + $tag = null; + $nerd = null; + $crowd = null; + if($this->params["dst_cti_model"]["as_name"]?->nerd != null){ + $nerd = $nerd . Html::tag('div', $this->params["dst_cti_model"]["as_name"]->nerd . " ", ['class' => "vertical-top"]); + } + if($this->params["dst_cti_model"]["as_num"]?->nerd != null){ + $nerd = $nerd . Html::tag('div', "(".$this->params["dst_cti_model"]["as_num"]->nerd.")", ['class' => "vertical-top"]); + } + if ($nerd){ + $nerd = $nerd ." ". Html::tag('p', '[NERD]', ['class' => "vertical-top gray"]) . "
    "; + } + if($this->params["dst_cti_model"]["as_name"]?->crowd != null){ + $crowd = $crowd . Html::tag('div', $this->params["dst_cti_model"]["as_name"]->crowd . " ", ['class' => "vertical-top"]); + } + if($this->params["dst_cti_model"]["as_num"]?->crowd != null){ + $crowd = $crowd . Html::tag('div', "(".$this->params["dst_cti_model"]["as_num"]->crowd.")", ['class' => "vertical-top"]); + } + if ($crowd){ + $crowd = $crowd . " ". Html::tag('p', '[Crowdsec]', ['class' => "vertical-top gray"]); + } + $tag = $nerd . $crowd; + if ($tag){ + return Html::tag('div', $tag); + } + } + ], + [ + "label" => "First seen", + 'format' => 'raw', + 'value' => function(){ + $tag = null; + if($this->params["dst_cti_model"]["first_seen"]?->nerd != null){ + $tag = $tag . Html::tag('div', $this->params["dst_cti_model"]["first_seen"]->nerd, ['class' => "vertical-top"]) ." ". Html::tag('p', '[NERD]', ['class' => "vertical-top gray"]) . "
    "; + } + if($this->params["dst_cti_model"]["first_seen"]?->crowd != null){ + $tag = $tag . Html::tag('div', $this->params["dst_cti_model"]["first_seen"]->crowd, ['class' => "vertical-top"]) ." ". Html::tag('p', '[Crowdsec]', ['class' => "vertical-top gray"]); + } + if ($tag){ + return Html::tag('div', $tag); + } + } + ], + [ + "label" => "Last seen", + 'format' => 'raw', + 'value' => function(){ + $tag = null; + if($this->params["dst_cti_model"]["last_seen"]?->nerd != null){ + $tag = $tag . Html::tag('div', $this->params["dst_cti_model"]["last_seen"]->nerd, ['class' => "vertical-top"]) ." ". Html::tag('p', '[NERD]', ['class' => "vertical-top gray"]) . "
    "; + } + if($this->params["dst_cti_model"]["last_seen"]?->crowd != null){ + $tag = $tag . Html::tag('div', $this->params["dst_cti_model"]["last_seen"]->crowd, ['class' => "vertical-top"]) ." ". Html::tag('p', '[Crowdsec]', ['class' => "vertical-top gray"]); + } + if ($tag){ + return Html::tag('div', $tag); + } + } + ], + [ + "label" => 'Last API update', + 'format' => 'raw', + 'value' => function(){ + $tag = null; + if($this->params["dst_cti_model"]['nerd'] != null){ + $tag = $tag . Html::tag('div', $this->params["dst_cti_model"]['nerd']->last_checked_at, ['class' => "vertical-top"]) ." ". Html::tag('p', '[NERD]', ['class' => "vertical-top gray"]) . "
    "; + } + if($this->params["dst_cti_model"]['crowdsec'] != null){ + $tag = $tag . Html::tag('div', $this->params["dst_cti_model"]['crowdsec']->last_checked_at, ['class' => "vertical-top"]) ." ". Html::tag('p', '[Crowdsec]', ['class' => "vertical-top gray"]); + } + if ($tag){ + return Html::tag('div', $tag); + } + } + ], + ], + ]) ?>
    +
    • +
  • laptopNetwork model information: params['dst_device']->description?>
    diff --git a/web/css/site.css b/web/css/site.css index afeff301..c3256df3 100755 --- a/web/css/site.css +++ b/web/css/site.css @@ -1,3 +1,27 @@ +.danger { + color: red; + text-shadow: -1px 0 black, 0 1px black, 1px 0 black, 0 -1px black; +} + +.warning { + color: yellow; + text-shadow: -1px 0 black, 0 1px black, 1px 0 black, 0 -1px black; +} + +.safe { + color: green; + text-shadow: -1px 0 black, 0 1px black, 1px 0 black, 0 -1px black; +} + +.vertical-top { + display: inline; + vertical-align: top; +} + +.gray { + color: gray; +} + .tooltip { background: #eee; box-shadow: 0 0 5px #999999; From 761306e2dc63695b909c88d619757b688e19b482 Mon Sep 17 00:00:00 2001 From: EEliseeva Date: Thu, 9 May 2024 11:22:13 +0200 Subject: [PATCH 02/10] added writing permissions for www-data --- secmon_preconfig.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/secmon_preconfig.sh b/secmon_preconfig.sh index b96a2c45..13e65ea0 100755 --- a/secmon_preconfig.sh +++ b/secmon_preconfig.sh @@ -110,6 +110,7 @@ echo -e "${GREEN}Done${NORMAL}" # Set 777 permissions so container secmon_app can write to directories chgrp www-data . chmod 777 ./web/assets/ +chmod 777 ./config/ chmod -R 777 ./rules/* || { echo -e "${RED}Changing access mode of the directory ./rules/* failed${NORMAL}" ; exit 1; } # Create lock file as a sign a config was run. From 59b16e40d0b7c46bb8c3c84fa37d6b8f7f9a6bfb Mon Sep 17 00:00:00 2001 From: EEliseeva Date: Sun, 12 May 2024 12:33:10 +0200 Subject: [PATCH 03/10] moved ip_rep.csv to new separate folder to better manage permissions --- .gitignore | 2 +- models/CtiModel.php | 4 ++-- secmon_preconfig.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index f7d19c07..b1a429b3 100755 --- a/.gitignore +++ b/.gitignore @@ -35,7 +35,7 @@ docker-compose.yml /config/anomaly_config.ini /config/secmon_config.yaml /config/cti_config.json -config/ip_rep.csv +/cti/ip_rep.csv /config/aggregator_config.ini /deployment/certificates /rules diff --git a/models/CtiModel.php b/models/CtiModel.php index a5249657..91a02072 100644 --- a/models/CtiModel.php +++ b/models/CtiModel.php @@ -160,7 +160,7 @@ public static function getRepFromCsv($ip) 'format' => Client::FORMAT_JSON ]]); - $filePath = "/var/www/html/secmon/config/ip_rep.csv"; + $filePath = "/var/www/html/secmon/cti/ip_rep.csv"; if (file_exists($filePath)) { $csv = file_get_contents($filePath); @@ -209,7 +209,7 @@ private static function getCsvData($client) if ($response->isOk) { //print("GOT CSV" . PHP_EOL); $csvData = $response->getContent(); - file_put_contents('/var/www/html/secmon/config/ip_rep.csv', $csvData); + file_put_contents('/var/www/html/secmon/cti/ip_rep.csv', $csvData); $csvData = str_getcsv($csvData, "\n", "", "#"); return $csvData; } diff --git a/secmon_preconfig.sh b/secmon_preconfig.sh index 13e65ea0..24f19297 100755 --- a/secmon_preconfig.sh +++ b/secmon_preconfig.sh @@ -110,7 +110,7 @@ echo -e "${GREEN}Done${NORMAL}" # Set 777 permissions so container secmon_app can write to directories chgrp www-data . chmod 777 ./web/assets/ -chmod 777 ./config/ +chmod 777 ./cti/ chmod -R 777 ./rules/* || { echo -e "${RED}Changing access mode of the directory ./rules/* failed${NORMAL}" ; exit 1; } # Create lock file as a sign a config was run. From 1ad6694e5b8168e42e9220dd99c86858fd89887d Mon Sep 17 00:00:00 2001 From: EEliseeva Date: Sun, 12 May 2024 12:49:07 +0200 Subject: [PATCH 04/10] fixed labeling in view --- models/CtiModel.php | 2 +- views/security-events/view.php | 16 ++++++---------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/models/CtiModel.php b/models/CtiModel.php index 91a02072..77c07bb0 100644 --- a/models/CtiModel.php +++ b/models/CtiModel.php @@ -94,7 +94,7 @@ public static function getCtiInfo($id) } $cti_row->reputation = isset($cti_row['nerd']) - ? $cti_row['nerd']['rep'] + ? round($cti_row['nerd']['rep'], 2) : CsvService::getRepFromCsv($cti_row['ip']); $cti_row->as_name = (object)["nerd" => $cti_row['nerd']['as_name'] ?? null, 'crowd' => $cti_row['crowdsec']['as_name'] ?? null]; diff --git a/views/security-events/view.php b/views/security-events/view.php index 5006cdf6..2cf99276 100644 --- a/views/security-events/view.php +++ b/views/security-events/view.php @@ -163,7 +163,7 @@ 'value' => $this->params["src_cti_model"]['crowdsec']->classification ?? null ], [ - "label" => "events", + "label" => "Recorded events", 'format' => 'raw', 'value' => function(){ $tag = null; @@ -232,7 +232,7 @@ } ], [ - "label" => "ip_range rep", + "label" => "IP range reputational score", 'format' => 'raw', 'value' => function(){ $tag = null; @@ -268,7 +268,7 @@ } ], [ - "label" => "AS name (ID)", + "label" => "AS name (AS number)", 'format' => 'raw', 'value' => function(){ $tag = null; @@ -423,10 +423,6 @@ "label" => 'IP', 'value' => $this->params["dst_cti_model"]['ip'] ], - [ - "label" => 'Reputation', - 'value' => $this->params["dst_cti_model"]['reputation'] - ], [ "label" => 'FMP', 'value' => $this->params["dst_cti_model"]['nerd']->fmp ?? null @@ -488,7 +484,7 @@ 'value' => $this->params["dst_cti_model"]['crowdsec']->classification ?? null ], [ - "label" => "events", + "label" => "Recorded events", 'format' => 'raw', 'value' => function(){ $tag = null; @@ -557,7 +553,7 @@ } ], [ - "label" => "ip_range rep", + "label" => "IP range reputational score", 'format' => 'raw', 'value' => function(){ $tag = null; @@ -593,7 +589,7 @@ } ], [ - "label" => "AS name (ID)", + "label" => "AS name (AS number)", 'format' => 'raw', 'value' => function(){ $tag = null; From 4575601917834c996368657b17af8b13dfdc9366 Mon Sep 17 00:00:00 2001 From: EEliseeva Date: Sun, 12 May 2024 13:05:00 +0200 Subject: [PATCH 05/10] obmedzenie pristupovych prav pre cti priecinok --- secmon_preconfig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/secmon_preconfig.sh b/secmon_preconfig.sh index 24f19297..773f7008 100755 --- a/secmon_preconfig.sh +++ b/secmon_preconfig.sh @@ -110,7 +110,7 @@ echo -e "${GREEN}Done${NORMAL}" # Set 777 permissions so container secmon_app can write to directories chgrp www-data . chmod 777 ./web/assets/ -chmod 777 ./cti/ +chmod 770 ./cti/ chmod -R 777 ./rules/* || { echo -e "${RED}Changing access mode of the directory ./rules/* failed${NORMAL}" ; exit 1; } # Create lock file as a sign a config was run. From 8cc54d863f45600b791b36f035f162219f117311 Mon Sep 17 00:00:00 2001 From: EEliseeva Date: Mon, 13 May 2024 10:16:23 +0200 Subject: [PATCH 06/10] git ignores empty folders, adding empty file to work around --- cti/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 cti/.gitkeep diff --git a/cti/.gitkeep b/cti/.gitkeep new file mode 100644 index 00000000..e69de29b From 330b2fdb099f8e9fd6f746d95baeda7ce807d299 Mon Sep 17 00:00:00 2001 From: Ekaterina Eliseeva Date: Mon, 13 May 2024 14:18:57 +0200 Subject: [PATCH 07/10] minor fixes --- commands/CtiController.php | 6 +++--- secmon_manager.py | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/commands/CtiController.php b/commands/CtiController.php index 0b754dd8..d68b4526 100644 --- a/commands/CtiController.php +++ b/commands/CtiController.php @@ -132,7 +132,7 @@ public function actionIndex() //print("PROCESSING SRC\n"); $src_cti_id = $this->processIp($srcIp, $whitelist, $connection, $client, $nerd_auth, $crowd_auth, $api_time_validity, $file_time_validity); $msg = str_replace("\n", "", $msg); - $msg = $msg . " src_cti_id=" . strval($src_cti_id); + $msg = $msg . " src_cti_id=" . strval($src_cti_id) . " "; } if ($dstIp != -1) { @@ -174,7 +174,7 @@ function processIp($ip, $whitelist, $connection, $client, $nerd_auth, $crowd_aut } ############# NERD ############# - if ($main[2] != null) { + if (($main[2] ?? null) != null) { //print("Pairing table has NERD table linked\n"); $object = $this->selectFromNERDTable($main[2], $connection, $ip, $client, $nerd_auth, $api_time_validity); } else { @@ -188,7 +188,7 @@ function processIp($ip, $whitelist, $connection, $client, $nerd_auth, $crowd_aut } ############# CROWD ############# - if ($main[1] != null) { + if (($main[1] ?? null) != null) { //print("Pairing table has CROWD table linked\n"); $object = $this->selectFromCROWDTable($main[1], $connection, $ip, $client, $crowd_auth, $api_time_validity); } else { diff --git a/secmon_manager.py b/secmon_manager.py index ecd982d0..3820a1cc 100755 --- a/secmon_manager.py +++ b/secmon_manager.py @@ -211,7 +211,6 @@ def get_config(): for module in secmon_config["ENRICHMENT"]: if module["enabled"]: ENABLED_ENRICHMENT_MODULES.append(module) - print(module["name"]) ALL_MODULES.append(module) return secmon_config From e05dede4dcd55db0939c5ee24901e8bd7366e0a6 Mon Sep 17 00:00:00 2001 From: Ekaterina Eliseeva Date: Tue, 14 May 2024 11:29:19 +0200 Subject: [PATCH 08/10] added missing false_pos property, minor fixes --- commands/CtiController.php | 34 +++++++++++++++--------------- views/security-events/view.php | 38 +++++++++++++++++++++++++++++----- 2 files changed, 50 insertions(+), 22 deletions(-) diff --git a/commands/CtiController.php b/commands/CtiController.php index d68b4526..8a79543b 100644 --- a/commands/CtiController.php +++ b/commands/CtiController.php @@ -134,7 +134,7 @@ public function actionIndex() $msg = str_replace("\n", "", $msg); $msg = $msg . " src_cti_id=" . strval($src_cti_id) . " "; } - + if ($dstIp != -1) { //print("PROCESSING DST\n"); $dst_cti_id = $this->processIp($dstIp, $whitelist, $connection, $client, $nerd_auth, $crowd_auth, $api_time_validity, $file_time_validity); @@ -180,7 +180,7 @@ function processIp($ip, $whitelist, $connection, $client, $nerd_auth, $crowd_aut } else { //print("Pairing table hasn't NERD table linked\n"); $object = $this->updateFromNERDapi($ip, $client, $nerd_auth); - if ($object != null && $object != -1) { + if ($object != null) { $nerd_id = $this->recordToNERDTable($object, $connection); $this->updatePairingTableNERD($connection, $main[0], $nerd_id); $main[2] = $nerd_id; @@ -194,7 +194,7 @@ function processIp($ip, $whitelist, $connection, $client, $nerd_auth, $crowd_aut } else { //print("Pairing table hasn't CROWD table linked\n"); $object = $this->updateFromCROWDapi($ip, $client, $crowd_auth); - if ($object != null && $object != -1) { + if ($object != null) { $crowd_id = $this->recordToCROWDTable($object, $connection); $this->updatePairingTableCROWD($connection, $main[0], $crowd_id); $main[1] = $crowd_id; @@ -306,10 +306,7 @@ function updateFromNERDapi($ip, $client, $nerd_auth) ->setUrl('https://nerd.cesnet.cz/nerd/api/v1/ip/' . (string)$ip . '/full') ->addHeaders(['Authorization' => $nerd_auth]) ->send(); - if ($nerd_response->statusCode == 404) { - //print("No NERD record found\n"); - return -1; - } elseif ($nerd_response->statusCode != 200) { + if ($nerd_response->statusCode != 200) { //print("NERD API call failed\n"); return null; } @@ -423,13 +420,13 @@ function updateCROWDTable($id, $object, $connection) behavior=$1, classification=$2, score_overall=$3, last_checked_at=$4, as_num=$5, as_name=$6, ip_range_24=$7, ip_range_24_rep=$8, geo_city=$9, geo_country=$10, - reverse_dns=$11, last_seen=$12, first_seen=$13 - WHERE id = $14', + reverse_dns=$11, last_seen=$12, first_seen=$13, false_pos=$14 + WHERE id = $15', array( $object->behavior, $object->classification, $object->score_overall, $object->crowd_timestamp, $object->crowd_AS_id, $object->crowd_AS_name, $object->crowd_ip_range, $object->crowd_ip_range_rep, $object->crowd_city, $object->crowd_country, - $object->crowd_reverse_dns, $object->crowd_last_seen, $object->crowd_first_seen, + $object->crowd_reverse_dns, $object->crowd_last_seen, $object->crowd_first_seen, $object->false_pos, $id ) ); @@ -448,10 +445,8 @@ function updateFromCROWDapi($ip, $client, $crowd_auth) ->setUrl('https://cti.api.crowdsec.net/v2/smoke/' . (string)$ip) ->addHeaders(['x-api-key' => $crowd_auth]) ->send(); - if ($crowd_response->statusCode == 404) { - //print("No CROWD record found\n"); - return -1; - } elseif ($crowd_response->statusCode != 200) { + + if ($crowd_response->statusCode != 200) { //print("CROWD API call failed\n"); return null; } @@ -466,7 +461,12 @@ function updateFromCROWDapi($ip, $client, $crowd_auth) foreach ($crowd_response->data["classifications"]["classifications"] as $list) { array_push($class, $list["label"]); } + $false_pos = array(); + foreach ($crowd_response->data["classifications"]["false_positives"] as $list) { + array_push($false_pos, $list["label"]); + } $object->classification = implode(", ", $class); + $object->false_pos = implode(", ", $false_pos); $object->score_overall = $crowd_response->data["scores"]["overall"]["total"]; $object->crowd_AS_id = $crowd_response->data["as_num"]; $object->crowd_AS_name = $crowd_response->data["as_name"]; @@ -491,17 +491,17 @@ function recordToCROWDTable($object, $connection) 'INSERT INTO cti_crowdsec(behavior, classification, score_overall, last_checked_at, as_num, as_name, ip_range_24, ip_range_24_rep, geo_city, geo_country, - reverse_dns, last_seen, first_seen) + reverse_dns, last_seen, first_seen, false_pos) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, - $11, $12, $13) + $11, $12, $13, $14) RETURNING id', array( $object->behavior, $object->classification, $object->score_overall, $object->crowd_timestamp, $object->crowd_AS_id, $object->crowd_AS_name, $object->crowd_ip_range, $object->crowd_ip_range_rep, $object->crowd_city, $object->crowd_country, - $object->crowd_reverse_dns, $object->crowd_last_seen, $object->crowd_first_seen + $object->crowd_reverse_dns, $object->crowd_last_seen, $object->crowd_first_seen, $object->false_pos ) ); $id = pg_fetch_row($result)[0]; diff --git a/views/security-events/view.php b/views/security-events/view.php index 2cf99276..a07672ac 100644 --- a/views/security-events/view.php +++ b/views/security-events/view.php @@ -160,7 +160,21 @@ ], [ "label" => 'Classification', - 'value' => $this->params["src_cti_model"]['crowdsec']->classification ?? null + 'format' => 'raw', + 'value' => function () { + if ($this->params["src_cti_model"]['crowdsec']?->classification === '' || $this->params["src_cti_model"]['crowdsec']?->classification === null) + return null; + return $this->params["src_cti_model"]['crowdsec']->classification ?? null; + } + ], + [ + "label" => 'False positives', + 'format' => 'raw', + 'value' => function () { + if ($this->params["src_cti_model"]['crowdsec']?->false_pos === '' || $this->params["src_cti_model"]['crowdsec']?->false_pos === null) + return null; + return $this->params["src_cti_model"]['crowdsec']->false_pos ?? null; + } ], [ "label" => "Recorded events", @@ -239,7 +253,7 @@ if($this->params["src_cti_model"]["ip_range_rep"]?->nerd != null){ $tag = $tag . Html::tag('div', $this->params["src_cti_model"]["ip_range_rep"]->nerd, ['class' => "vertical-top"]) ." ". Html::tag('p', '[NERD]', ['class' => "vertical-top gray"]) . "
    "; } - if($this->params["src_cti_model"]["ip_range_rep"]?->crowd != null){ + if($this->params["src_cti_model"]["ip_range_rep"]?->crowd != null && $this->params["src_cti_model"]["ip_range_rep"]?->crowd !== ' '){ $tag = $tag . Html::tag('div', $this->params["src_cti_model"]["ip_range_rep"]->crowd, ['class' => "vertical-top"]) ." ". Html::tag('p', '[Crowdsec]', ['class' => "vertical-top gray"]); } if ($tag){ @@ -464,7 +478,7 @@ $reputationNerdDiv = null; if ($reputationNerd !== null) { $reputationNerdDiv = Html::tag('i', $iconNameNerd, ['class' => "material-icons vertical-top " . $colorClassNerd]) ." ". Html::tag('div', $reputationNerd, ['class' => "vertical-top"]) . " " . Html::tag('p', '[NERD]', ['class' => "vertical-top gray"]) . "
    "; - } + } $scoreCrowdDiv = null; if ($scoreCrowd !== null){ $scoreCrowdDiv = Html::tag('i', $iconNameCrowd, ['class' => "material-icons vertical-top " . $colorClassCrowd]) ." ". Html::tag('div', $scoreCrowd, ['class' => "vertical-top"]) . " " . Html::tag('p', '[Crowdsec]', ['class' => "vertical-top gray"]); @@ -481,7 +495,21 @@ ], [ "label" => 'Classification', - 'value' => $this->params["dst_cti_model"]['crowdsec']->classification ?? null + 'format' => 'raw', + 'value' => function () { + if ($this->params["dst_cti_model"]['crowdsec']?->classification === '' || $this->params["dst_cti_model"]['crowdsec']?->classification === null) + return null; + return $this->params["dst_cti_model"]['crowdsec']->classification ?? null; + } + ], + [ + "label" => 'False positives', + 'format' => 'raw', + 'value' => function () { + if ($this->params["dst_cti_model"]['crowdsec']?->false_pos === '' || $this->params["dst_cti_model"]['crowdsec']?->false_pos === null) + return null; + return $this->params["dst_cti_model"]['crowdsec']->false_pos ?? null; + } ], [ "label" => "Recorded events", @@ -560,7 +588,7 @@ if($this->params["dst_cti_model"]["ip_range_rep"]?->nerd != null){ $tag = $tag . Html::tag('div', $this->params["dst_cti_model"]["ip_range_rep"]->nerd, ['class' => "vertical-top"]) ." ". Html::tag('p', '[NERD]', ['class' => "vertical-top gray"]) . "
    "; } - if($this->params["dst_cti_model"]["ip_range_rep"]?->crowd != null){ + if($this->params["dst_cti_model"]["ip_range_rep"]?->crowd != null && $this->params["dst_cti_model"]["ip_range_rep"]?->crowd !== ' '){ $tag = $tag . Html::tag('div', $this->params["dst_cti_model"]["ip_range_rep"]->crowd, ['class' => "vertical-top"]) ." ". Html::tag('p', '[Crowdsec]', ['class' => "vertical-top gray"]); } if ($tag){ From dc4dca5602006429d0c3983bac9612205df426da Mon Sep 17 00:00:00 2001 From: Ekaterina Eliseeva Date: Wed, 15 May 2024 17:53:47 +0200 Subject: [PATCH 09/10] view bug fix --- views/security-events/view.php | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/views/security-events/view.php b/views/security-events/view.php index a07672ac..34232086 100644 --- a/views/security-events/view.php +++ b/views/security-events/view.php @@ -156,7 +156,12 @@ ], [ "label" => 'Blacklists', - 'value' => $this->params["src_cti_model"]['nerd']->blacklists ?? null + 'format' => 'raw', + 'value' => function () { + if ($this->params["src_cti_model"]['nerd']?->blacklists === '' || $this->params["src_cti_model"]['nerd']?->blacklists === null) + return null; + return $this->params["src_cti_model"]['nerd']->blacklists ?? null; + } ], [ "label" => 'Classification', @@ -491,7 +496,12 @@ ], [ "label" => 'Blacklists', - 'value' => $this->params["dst_cti_model"]['nerd']->blacklists ?? null + 'format' => 'raw', + 'value' => function () { + if ($this->params["dst_cti_model"]['nerd']?->blacklists === '' || $this->params["dst_cti_model"]['nerd']?->blacklists === null) + return null; + return $this->params["dst_cti_model"]['nerd']->blacklists ?? null; + } ], [ "label" => 'Classification', From 581c8a247d8b2827ddda07247c285b195c8a2d44 Mon Sep 17 00:00:00 2001 From: Ekaterina Eliseeva Date: Wed, 22 May 2024 12:07:43 +0200 Subject: [PATCH 10/10] minor fixes --- commands/CtiController.php | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/commands/CtiController.php b/commands/CtiController.php index 8a79543b..eadce906 100644 --- a/commands/CtiController.php +++ b/commands/CtiController.php @@ -127,7 +127,7 @@ public function actionIndex() $dstIp = substr($msg, $position1 + strlen("dst="), $position3); } $connection = pg_connect("host=" . $host . " dbname=" . $database . " user=" . $user . " password=" . $password); - + if ($srcIp != -1) { //print("PROCESSING SRC\n"); $src_cti_id = $this->processIp($srcIp, $whitelist, $connection, $client, $nerd_auth, $crowd_auth, $api_time_validity, $file_time_validity); @@ -230,12 +230,13 @@ function recordToPairingTable($ip, $connection) if ($ip != -1) { //print("IP is defined\n"); $result = pg_query_params($connection, 'INSERT INTO cti(ip) VALUES ($1) RETURNING id', array($ip)); - $id = pg_fetch_row($result)[0]; + if ($result == false) { //print("Query failed\n"); return -1; } //print("Successfuly inserted\n"); + $id = pg_fetch_row($result)[0]; return $id; } } @@ -291,7 +292,6 @@ function updateNERDTable($id, $object, $connection) $id ) ); - $id = pg_fetch_row($result)[0]; if ($result == false) { //print("Query failed\n"); } @@ -318,9 +318,10 @@ function updateFromNERDapi($ip, $client, $nerd_auth) array_push($bl, $list["name"]); } $object->blacklists = implode(", ", $bl); + $object->blacklists = substr($object->blacklists, 0, 254); $object->rep = $nerd_response->data["rep"]; - $object->nerd_AS_id = $nerd_response->data["asn"][0]["_id"]; - $object->nerd_AS_name = $nerd_response->data["asn"][0]["name"]; + $object->nerd_AS_id = $nerd_response->data["asn"][0]["_id"] ?? null; + $object->nerd_AS_name = $nerd_response->data["asn"][0]["name"] ?? null; $object->nerd_ip_range = $nerd_response->data["bgppref"]["_id"]; $object->nerd_ip_range_rep = $nerd_response->data["bgppref"]["rep"]; @@ -329,6 +330,7 @@ function updateFromNERDapi($ip, $client, $nerd_auth) array_push($events_cat, $list["cat"]); } $object->events = implode(", ", array_unique($events_cat)); + $object->events = substr($object->events, 0, 254); $object->nerd_city = $nerd_response->data["geo"]["city"]; $object->nerd_country = $nerd_response->data["geo"]["ctry"]; $object->nerd_hostname = $nerd_response->data["hostname"]; @@ -361,11 +363,12 @@ function recordToNERDTable($object, $connection) $object->nerd_hostname, $object->nerd_last_activity, $object->nerd_first_activity ) ); - $id = pg_fetch_row($result)[0]; + if ($result == false) { //print("Query failed\n"); return -1; } + $id = pg_fetch_row($result)[0]; return $id; } @@ -373,7 +376,7 @@ function updatePairingTableNERD($connection, $main_id, $nerd_id) { //print("updatePairingTableNERD started\n"); $result = pg_query_params($connection, 'UPDATE cti SET fk_nerd_id = $1 WHERE id = $2 RETURNING id', array($nerd_id, $main_id)); - $id = pg_fetch_row($result)[0]; + if ($result == false) { //print("Query failed\n"); } @@ -430,7 +433,7 @@ function updateCROWDTable($id, $object, $connection) $id ) ); - $id = pg_fetch_row($result)[0]; + if ($result == false) { //print("Query failed\n"); } @@ -504,11 +507,12 @@ function recordToCROWDTable($object, $connection) $object->crowd_reverse_dns, $object->crowd_last_seen, $object->crowd_first_seen, $object->false_pos ) ); - $id = pg_fetch_row($result)[0]; + if ($result == false) { //print("Query failed\n"); return -1; } + $id = pg_fetch_row($result)[0]; return $id; } @@ -516,7 +520,7 @@ function updatePairingTableCROWD($connection, $main_id, $crowd_id) { //print("updatePairingTableCROWD started\n"); $result = pg_query_params($connection, 'UPDATE cti SET fk_crowdsec_id = $1 WHERE id = $2 RETURNING id', array($crowd_id, $main_id)); - $id = pg_fetch_row($result)[0]; + if ($result == false) { //print("Query failed\n"); }