commit e128813e303266e8395ec6927a51052543bf3a60 Author: Guillaume Lapierre Date: Mon Mar 25 18:28:56 2024 +0100 Initial release diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..deb01bf --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +conf/report-parser.conf +data/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..0cabb11 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,71 @@ +## https://github.com/userjack6880/Open-Report-Parser +FROM debian:12 AS parserbuild + +RUN apt-get update +RUN apt-get -y upgrade +RUN apt-get install -y git + +WORKDIR /src +RUN git clone https://github.com/userjack6880/Open-Report-Parser.git +RUN mkdir -p /opt/open-report-parser + +# Extract archive +WORKDIR /src/Open-Report-Parser +RUN git archive --format tar main | tar -x -C /opt/open-report-parser + +## https://github.com/userjack6880/Open-DMARC-Analyzer + +### We need composer + +FROM composer:latest AS composer + +FROM php:cli AS analyzerbuilder +RUN apt-get update +RUN apt-get -y upgrade +RUN apt-get install -y git +RUN mkdir -p /src/analyzer + +# Sénat +ADD http://pki.senat.fr/pki/CARootSenat.crt /usr/local/share/ca-certificates/CARootSenat.crt +RUN update-ca-certificates -v + +COPY --from=composer /usr/bin/composer /usr/bin/composer + +WORKDIR /src +RUN git clone https://github.com/userjack6880/Open-DMARC-Analyzer.git + +WORKDIR /src/Open-DMARC-Analyzer +RUN git archive --format tar version-1 | tar -x -C /src/analyzer + +WORKDIR /src/analyzer +RUN /usr/bin/composer require "kevinoo/phpwhois":"^6.3" + +## The Image! +FROM php:8.3-apache + +# Install dependencies +RUN apt-get update +RUN apt-get -y upgrade + +RUN apt-get install -y libpq-dev libfile-mimeinfo-perl libmail-imapclient-perl libmime-tools-perl \ + libxml-simple-perl libio-socket-inet6-perl libio-socket-ip-perl libperlio-gzip-perl \ + libmail-mbox-messageparser-perl libwww-perl unzip \ + libdbd-mysql-perl libdbd-pg-perl \ + liblwp-protocol-https-perl libencode-perl libtime-piece-mysql-perl \ + libjson-perl + +RUN docker-php-ext-install pdo pdo_mysql pdo_pgsql + +# open-report-parser +COPY --from=parserbuild /opt/open-report-parser /opt/open-report-parser + +# open-report-analyzer +COPY --chown=33:33 --from=analyzerbuilder /src/analyzer /var/www/html + +# open-report-analyzer configuration +COPY --chown=33:33 ./conf/config.analyzer.php /var/www/html/config.php + +# configure PHP +RUN cd /usr/local/etc/php ; \ + mv php.ini-production php.ini ; \ + rm php.ini-development diff --git a/README.md b/README.md new file mode 100644 index 0000000..b7312cc --- /dev/null +++ b/README.md @@ -0,0 +1,54 @@ +# Open Report Analyzer and Parser + +This docker-compose file aims to provide both Open Report Analyzer and Open Report Parser from [userjack6880](https://github.com/userjack6880) inside the same container. + +On first run you will need to create the required browsing `http://localhost:8080/install.php` + +## Sample docker-compose file + +Sample file, you will need to adapt it to your configuration! + +## Open Report Parser + +open report parser is a perl based tool to parse DMARC reports. Please see [here](https://github.com/userjack6880/Open-Report-Parser) for a full description of the tool. + +You will need one of those: +* a MariaDB 10.5 or equivalent database +* PostgreSQL 13.9+ + +You need to setup a configuration file and put it inside `/opt/open-report-parser` + +Syntax for folders is setup dependant. For my cyrusd-imap I had to use `INBOX/dmarc` syntax (folders not flattened). + +To run the parser: + +``` +docker compose run --rm -it open-report-analyzer /bin/sh -c 'cd /opt/open-report-parser; ./report-parser.pl -i --info' +``` + +You will want to add this to a cron job which is out of the scope of this readme file! + +## Open Report Analyzer + +Default configuration file has beed modified to read environment variables from docker. You can +user `_FILE` suffix to read the value from a file. See docker compose secrets! + +Apache will listen on port 80 + +Environment variables and default values: + +| Environment variable | Meaning | Default value | +|----------------------|---------------------------------------------------------------------------------------------------------------------------|----------------------------| +| DB_HOST | Database hostname | `localhost` | +| DB_USER | Database username | `dmarc` | +| DB_PASS | Database password | `password` | +| DB_NAME | Database name | `dmarc` | +| DB_PORT | Default port 3306, 5432 for pgsql | `3306` | +| DB_TYPE | supported mysql and pgsql | `mysql` | +| DEBUG | not currently used! | `1` | +| TEMPLATE | available openda and openda_light | `openda` | +| AUTO_LOADER | should not need to change this! | `vendor/autoload.php` | +| GEO_ENABLE | see [official documentation](https://github.com/userjack6880/Open-DMARC-Analyzer) | `1` | +| GEO_DB | Path to the MaxMind GeoIP database (not provided) | `includes/geolite2.mmdb` | +| DATE_RANGE | Standard starting date range for data presented. Valid date signifiers are `m`, `w` and `d` for "month", "week" and "day" | `-1w` | + diff --git a/conf/config.analyzer.php b/conf/config.analyzer.php new file mode 100644 index 0000000..3381840 --- /dev/null +++ b/conf/config.analyzer.php @@ -0,0 +1,76 @@ +. +---------------------------------------------------------------------------- */ + + // Database Settings + define('DB_HOST', getEnvOrDefault('DB_HOST','localhost')); + define('DB_USER', getEnvOrDefault('DB_USER','dmarc')); + define('DB_PASS', getEnvOrDefault('DB_PASS','password')); + define('DB_NAME', getEnvOrDefault('DB_NAME','dmarc')); + define('DB_PORT', getEnvOrDefault('DB_PORT','3306')); // default port 3306, 5432 for pgsql + define('DB_TYPE', getEnvOrDefault('DB_TYPE','mysql')); // supported mysql and pgsql + + // Debug Settings + define('DEBUG', getEnvOrDefault('DEBUG','1', true)); + + // Template Settings + define('TEMPLATE', getEnvOrDefault('TEMPLATE','openda')); + + // Package Loader + define('AUTO_LOADER', getEnvOrDefault('AUTO_LOADER', 'vendor/autoload.php')); // autoloader for composer installed libraries + + // GeoIP2 Settings + define('GEO_ENABLE', getEnvOrDefault('GEO_ENABLE','1',true)); // 0 - disable GeoIP2, 1 - enable GeoIP2 + define('GEO_DB', getEnvOrDefault('GEO_DB','includes/geolite2.mmdb')); // location of GeoIP2 database + + // Date Range + define('DATE_RANGE', getEnvOrDefault('DATE_RANGE', '-1w')); + + // Get value from environment or use default value if not provided + // will also test _FILE to get secrets if needed! + function getEnvOrDefault($envName, $defaultValue, $isInt = false) { + $value = getenv($envName . "_FILE"); + + if ($value !== false) { + // Get value from file! + $value = file_get_contents($value); + } else { + $value = getenv($envName); + } + + // Still no value from environment or error while reading file? + if ($value === false) { + $value = $defaultValue; + } + + if ($isInt) { + $value = intval($value, 10); + } + + return $value; + } + +?> \ No newline at end of file diff --git a/conf/report-parser.conf.pub b/conf/report-parser.conf.pub new file mode 100644 index 0000000..61dde2b --- /dev/null +++ b/conf/report-parser.conf.pub @@ -0,0 +1,88 @@ +# ----------------------------------------------------------------------------- +# +# Open Report Parser - Open Source report parser +# Copyright (C) 2023 John Bradley (userjack6880) +# Copyright (C) 2016 TechSneeze.com +# Copyright (C) 2012 John Bieling +# +# report-parser.conf +# configuration file +# +# Available at: https://github.com/userjack6880/Open-Report-Parser +# +# ----------------------------------------------------------------------------- +# +# This file is part of Open Report Parser. +# +# Open Report Parser is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# this program. If not, see . +# +# ----------------------------------------------------------------------------- + +# If IMAP access is not used, config options starting with $imap do not need to +# be set and are ignored. + +$debug = 0; +$delete_reports = 0; +#$dmarc_only = 0; +# if set to 1, do not process tls reports, if set to -1 do not process +# dmarc reports - defaults to 1 for legacy support +# this is ignored for all methods except IMAP, use --tls to process +# TLS reports for other methods + +#$dbtype = 'mysql'; # Supported types - mysql, postgres - defaults to mysql if unset +$dbname = 'dmarc'; +$dbuser = 'dmarc'; +$dbpass = 'password'; +$dbhost = 'db'; # Set the hostname if we can't connect to the local socket. +$dbport = '3306'; + +$imapserver = 'imap.server'; +$imapuser = 'username'; +$imappass = 'password'; +$imapport = '143'; +$imapssl = '0'; # If set to 1, remember to change server port to 993 and disable imaptls. +$imaptls = '0'; +$tlsverify = '0'; +$imapignoreerror = '0'; # recommended if you use MS Exchange 2007, ... +#$imapauth = 'simple'; # supported - simple, oauth2 - defaults to simple if unset + +# see documentation for detailed setup +#$oauthclientid = ''; +#$oauthuri = ''; + +$imapdmarcfolder = 'dmarc'; +$imaptlsfolder = 'tls'; + +# If $imapxxxproc is set, processed IMAP messages will be moved (overruled by +# the --delete option!) +# $imapdmarcproc = 'dmarc.Processed'; +# $imaptlsproc = 'tls.Processed'; + +# If $imapxxxerr is set, IMAP messages that fail will be moved. If unset, failed messages +# will move to $imapdmarcproc (if it is set). Overruled by the --delete option! +# $imapdmarcerr = 'dmarc.notProcessed'; +# $imaptlserr = 'tls.notProcessed'; + +# maximum size of XML/JSON files to store in database, long files can cause transaction aborts +$maxsize_xml = 50000; +$maxsize_json = 50000; + +# store XML/JSON as base64 encopded gzip in database (save space, harder usable) +$compress_xml = 0; +$compress_json = 0; + +# if there was an error during file processing (message does not contain XML or ZIP parts, +# or a database error) the parser reports an error and does not delete the file, even if +# delete_reports is set (or --delete is given). Deletion can be enforced by delete_failed, +# however not for database errors. +$delete_failed = 0; \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..57bb0a8 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,32 @@ +services: + open-report-analyzer: + build: . + image: zeguigui/open-report-analyzer:latest + restart: always + environment: + - "DB_HOST=db" + - "DB_USER=dmarc" + - "DB_PASS=password" + - "DB_NAME=dmarc" + - "DB_PORT=3306" + - "DB_TYPE=mysql" + - "GEO_ENABLE=0" + - "DATE_RANGE=-1w" + volumes: + - ./conf/report-parser.conf:/opt/open-report-parser/report-parser.conf:ro + ports: + - 8080:80 + depends_on: + - db + + db: + image: mariadb:lts + restart: always + environment: + - "MARIADB_RANDOM_ROOT_PASSWORD=yes" + - "MARIADB_DATABASE=dmarc" + - "MARIADB_USER=dmarc" + - "MARIADB_PASSWORD=password" + volumes: + - ./data:/var/lib/mysql + - ./init.sql:/docker-entrypoint-initdb.d/init.sql diff --git a/init.sql b/init.sql new file mode 100644 index 0000000..733b6fe --- /dev/null +++ b/init.sql @@ -0,0 +1,100 @@ +-- Adminer 4.8.1 MySQL 10.11.6-MariaDB-0+deb12u1 dump + +SET NAMES utf8; +SET time_zone = '+00:00'; +SET foreign_key_checks = 0; +SET sql_mode = 'NO_AUTO_VALUE_ON_ZERO'; + +SET NAMES utf8mb4; + +CREATE TABLE `oauth` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `access_token` varchar(4096) DEFAULT NULL, + `refresh_token` varchar(1024) DEFAULT NULL, + `expire` timestamp NOT NULL, + `valid` int(10) unsigned NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; + + +CREATE TABLE `report` ( + `serial` int(10) unsigned NOT NULL AUTO_INCREMENT, + `mindate` timestamp NOT NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(), + `maxdate` timestamp NULL DEFAULT NULL, + `domain` varchar(255) NOT NULL, + `org` varchar(255) NOT NULL, + `reportid` varchar(255) NOT NULL, + `email` varchar(255) DEFAULT NULL, + `extra_contact_info` varchar(255) DEFAULT NULL, + `policy_adkim` varchar(20) DEFAULT NULL, + `policy_aspf` varchar(20) DEFAULT NULL, + `policy_p` varchar(20) DEFAULT NULL, + `policy_sp` varchar(20) DEFAULT NULL, + `policy_pct` tinyint(3) unsigned DEFAULT NULL, + `raw_xml` mediumtext DEFAULT NULL, + PRIMARY KEY (`serial`), + UNIQUE KEY `domain` (`domain`,`reportid`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci ROW_FORMAT=COMPRESSED; + + +CREATE TABLE `report_stats` (`serial` int(10) unsigned, `domain` varchar(255), `rcount` int(10) unsigned, `disposition` enum('none','quarantine','reject','unknown'), `reason` varchar(255), `policy_p` varchar(20), `policy_pct` tinyint(3) unsigned, `dkimdomain` varchar(255), `dkimresult` enum('none','pass','fail','neutral','policy','temperror','permerror','unknown'), `dkim_align` enum('fail','pass','unknown'), `spfdomain` varchar(255), `spfresult` enum('none','neutral','pass','fail','softfail','temperror','permerror','unknown'), `spf_align` enum('fail','pass','unknown'), `mindate` timestamp, `maxdate` timestamp); + + +CREATE TABLE `rptrecord` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `serial` int(10) unsigned NOT NULL, + `ip` int(10) unsigned DEFAULT NULL, + `ip6` binary(16) DEFAULT NULL, + `rcount` int(10) unsigned NOT NULL, + `disposition` enum('none','quarantine','reject','unknown') DEFAULT NULL, + `reason` varchar(255) DEFAULT NULL, + `dkimdomain` varchar(255) DEFAULT NULL, + `dkimresult` enum('none','pass','fail','neutral','policy','temperror','permerror','unknown') DEFAULT NULL, + `spfdomain` varchar(255) DEFAULT NULL, + `spfresult` enum('none','neutral','pass','fail','softfail','temperror','permerror','unknown') DEFAULT NULL, + `spf_align` enum('fail','pass','unknown') NOT NULL, + `dkim_align` enum('fail','pass','unknown') NOT NULL, + `identifier_hfrom` varchar(255) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `serial` (`serial`,`ip`), + KEY `serial6` (`serial`,`ip6`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; + + +CREATE TABLE `tls` ( + `serial` int(10) unsigned NOT NULL AUTO_INCREMENT, + `mindate` timestamp NOT NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(), + `maxdate` timestamp NULL DEFAULT NULL, + `domain` varchar(255) NOT NULL, + `org` varchar(255) NOT NULL, + `reportid` varchar(255) NOT NULL, + `email` varchar(255) DEFAULT NULL, + `policy_mode` varchar(20) DEFAULT NULL, + `summary_success` int(11) DEFAULT NULL, + `summary_failure` int(11) DEFAULT NULL, + `raw_json` mediumtext DEFAULT NULL, + PRIMARY KEY (`serial`), + UNIQUE KEY `domain` (`domain`,`reportid`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci ROW_FORMAT=COMPRESSED; + + +CREATE TABLE `tlsrecord` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `serial` int(10) unsigned NOT NULL, + `send_ip` int(10) unsigned DEFAULT NULL, + `send_ip6` binary(16) DEFAULT NULL, + `recv_ip` int(10) unsigned DEFAULT NULL, + `recv_ip6` binary(16) DEFAULT NULL, + `recv_mx` varchar(255) DEFAULT NULL, + `type` varchar(255) DEFAULT NULL, + `count` int(10) unsigned NOT NULL, + PRIMARY KEY (`id`), + KEY `serial` (`serial`,`send_ip`), + KEY `serial6` (`serial`,`send_ip6`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; + + +DROP TABLE IF EXISTS `report_stats`; +CREATE ALGORITHM=UNDEFINED DEFINER=`dmarc`@`%` SQL SECURITY DEFINER VIEW `report_stats` AS (select `report`.`serial` AS `serial`,`report`.`domain` AS `domain`,`rptrecord`.`rcount` AS `rcount`,`rptrecord`.`disposition` AS `disposition`,`rptrecord`.`reason` AS `reason`,`report`.`policy_p` AS `policy_p`,`report`.`policy_pct` AS `policy_pct`,`rptrecord`.`dkimdomain` AS `dkimdomain`,`rptrecord`.`dkimresult` AS `dkimresult`,`rptrecord`.`dkim_align` AS `dkim_align`,`rptrecord`.`spfdomain` AS `spfdomain`,`rptrecord`.`spfresult` AS `spfresult`,`rptrecord`.`spf_align` AS `spf_align`,`report`.`mindate` AS `mindate`,`report`.`maxdate` AS `maxdate` from (`rptrecord` left join `report` on(`report`.`serial` = `rptrecord`.`serial`))); + +-- 2024-03-25 17:18:03