Revert opendmarc to use mysql

An earlier commit started transitioning opendmarc to use postgres, but
this was incomplete.  This patch reverts that change and uses mysql for
the reporting database.

Other changes:

* Do not maintain a copy of the database import schema.  A copy is
  included in the distribution in /usr/share/doc, so that is used
  instead.
* The configuration file is replaced with the distribution's sample
  configuration.  A second patch will restore the actual configuration.
  This will make the changes easier to see if the default configuraton
  file changes in future versions of opendmarc.

roles/mailserver/files/etc_opendmarc_import.sql

@@ -1,106 +0,0 @@
-# Source: http://www.trusteddomain.org/pipermail/opendmarc-users/2015-February/000447.html
-
-START TRANSACTION;
-
-SET standard_conforming_strings=off;
-SET escape_string_warning=off;
-SET CONSTRAINTS ALL DEFERRED;
-
-CREATE TABLE "domains" (
-  "id" integer NOT NULL,
-  "name" varchar(510) NOT NULL,
-  "firstseen" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
-  PRIMARY KEY ("id"),
-  UNIQUE ("name")
-);
-
-CREATE TABLE "ipaddr" (
-  "id" integer NOT NULL,
-  "addr" varchar(128) NOT NULL,
-  "firstseen" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
-  PRIMARY KEY ("id"),
-  UNIQUE ("addr")
-);
-
-CREATE TABLE "messages" (
-  "id" integer NOT NULL,
-  "date" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
-  "jobid" varchar(256) NOT NULL,
-  "reporter" integer  NOT NULL,
-  "policy" tinyint(3)  NOT NULL,
-  "disp" tinyint(3)  NOT NULL,
-  "ip" integer  NOT NULL,
-  "env_domain" integer  NOT NULL,
-  "from_domain" integer  NOT NULL,
-  "policy_domain" integer  NOT NULL,
-  "spf" tinyint(3)  NOT NULL,
-  "align_dkim" tinyint(3)  NOT NULL,
-  "align_spf" tinyint(3)  NOT NULL,
-  "sigcount" tinyint(3)  NOT NULL,
-  PRIMARY KEY ("id"),
-  UNIQUE ("reporter", "date", "jobid")
-);
-
-CREATE TABLE "reporters" (
-  "id" integer NOT NULL,
-  "name" varchar(510) NOT NULL,
-  "firstseen" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
-  PRIMARY KEY ("id"),
-  UNIQUE ("name")
-);
-
-CREATE TABLE "requests" (
-  "id" integer NOT NULL,
-  "domain" integer NOT NULL,
-  "repuri" varchar(510) NOT NULL,
-  "adkim" tinyint(4) NOT NULL,
-  "aspf" tinyint(4) NOT NULL,
-  "policy" tinyint(4) NOT NULL,
-  "spolicy" tinyint(4) NOT NULL,
-  "pct" tinyint(4) NOT NULL,
-  "locked" tinyint(4) NOT NULL,
-  "firstseen" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
-  "lastsent" timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
-  PRIMARY KEY ("id"),
-  UNIQUE ("domain")
-);
-
-CREATE TABLE "signatures" (
-  "id" integer NOT NULL,
-  "message" integer NOT NULL,
-  "domain" integer NOT NULL,
-  "pass" tinyint(4) NOT NULL,
-  "error" tinyint(4) NOT NULL,
-  PRIMARY KEY ("id")
-);
-
-COMMIT;
-
-START TRANSACTION;
-
-CREATE SEQUENCE domains_id_seq;
-SELECT setval('domains_id_seq', max(id)) FROM domains;
-ALTER TABLE "domains" ALTER COLUMN "id" SET DEFAULT nextval('domains_id_seq');
-
-CREATE SEQUENCE ipaddr_id_seq;
-SELECT setval('ipaddr_id_seq', max(id)) FROM ipaddr;
-ALTER TABLE "ipaddr" ALTER COLUMN "id" SET DEFAULT nextval('ipaddr_id_seq');
-
-CREATE SEQUENCE messages_id_seq;
-SELECT setval('messages_id_seq', max(id)) FROM messages;
-ALTER TABLE "messages" ALTER COLUMN "id" SET DEFAULT nextval('messages_id_seq');
-
-CREATE SEQUENCE reporters_id_seq;
-SELECT setval('reporters_id_seq', max(id)) FROM reporters;
-ALTER TABLE "reporters" ALTER COLUMN "id" SET DEFAULT nextval('reporters_id_seq');
-
-CREATE SEQUENCE requests_id_seq;
-SELECT setval('requests_id_seq', max(id)) FROM requests;
-ALTER TABLE "requests" ALTER COLUMN "id" SET DEFAULT nextval('requests_id_seq');
-
-CREATE SEQUENCE signatures_id_seq;
-SELECT setval('signatures_id_seq', max(id)) FROM signatures;
-ALTER TABLE "signatures" ALTER COLUMN "id" SET DEFAULT nextval('signatures_id_seq');
-
-COMMIT;

roles/mailserver/handlers/main.yml

@@ -14,10 +14,6 @@
   action: shell PGPASSWORD='{{ mail_db_password }}' psql -h localhost -d {{ mail_db_database }} -U {{ mail_db_username }} -f /etc/postfix/import.sql --set ON_ERROR_STOP=1
   notify: restart postfix
 
-- name: import sql opendmarc
-  action: shell PGPASSWORD='{{ mail_db_opendmarc_password }}' psql -h localhost -d {{ mail_db_opendmarc_database }} -U {{ mail_db_opendmarc_username }} -f /etc/opendmarc/import.sql --set ON_ERROR_STOP=1
-  notify: restart postfix
-
 - name: restart opendmarc
   service: name=opendmarc state=restarted
 

roles/mailserver/tasks/opendmarc.yml

@@ -1,8 +1,8 @@
 - name: Install OpenDMARC milter and related packages
   apt: pkg={{ item }} state=installed update_cache=yes
   with_items:
-    - postgresql
-    - python-psycopg2
+    - mysql-server
+    - python-mysqldb
     - opendmarc
 
 - name: Copy OpenDMARC configuration file into place
@@ -22,14 +22,14 @@
     - restart postfix
 
 - name: Create database user for OpenDMARC reports
-  postgresql_user: login_host=localhost login_user={{ db_admin_username }} login_password="{{ db_admin_password }}" name={{ mail_db_opendmarc_username }} password="{{ mail_db_opendmarc_password }}" state=present
+  mysql_user: user={{ mail_db_opendmarc_username }} password={{ mail_db_opendmarc_password }} state=present priv="opendmarc.*:ALL"
 
 - name: Create database for OpenDMARC reports
-  postgresql_db: login_host=localhost login_user={{ db_admin_username }} login_password="{{ db_admin_password }}" name={{ mail_db_opendmarc_database }} state=present owner={{ mail_db_opendmarc_username }}
+  mysql_db: name={{ mail_db_opendmarc_database }} state=present
 
-- name: Copy OpenDMARC database schema file into place
-  copy: src=etc_opendmarc_import.sql dest=/etc/opendmarc/import.sql owner=root group=root mode=0600
-  notify: import sql opendmarc
+- name: Import database schema for OpenDMARC reports
+  mysql_db: name={{ mail_db_opendmarc_database }} state=import target=/usr/share/doc/opendmarc/schema.mysql
+  tags: import_mysql_postfix
 
 - name: Copy nightly OpenDMARC report generation script into place
   template: src=etc_opendmarc_report.sh.j2 dest=/etc/opendmarc/report.sh owner=root group=root mode="755"

roles/mailserver/templates/etc_opendmarc.conf.j2

@@ -1,41 +1,336 @@
-# This is a basic configuration that can easily be adapted to suit a standard
-# installation. For more advanced options, see opendkim.conf(5) and/or
-# /usr/share/doc/opendmarc/examples/opendmarc.conf.sample.
+##
+## opendmarc.conf -- configuration file for OpenDMARC filter
+##
+## Copyright (c) 2012-2014, The Trusted Domain Project.  All rights reserved.
+##
 
 ##  AuthservID (string)
-##      defaults to MTA name
+##  	defaults to MTA name
+##
+##  Sets the "authserv-id" to use when generating the Authentication-Results:
+##  header field after verifying a message.  If the string "HOSTNAME" is
+##  provided, the name of the host running the filter (as returned by the
+##  gethostname(3) function) will be used.  
+#
+# AuthservID name
+
+##  AuthservIDWithJobID { true | false }
+##  	default "false"
+##
+##  If "true", requests that the authserv-id portion of the added
+##  Authentication-Results header fields contain the job ID of the message
+##  being evaluated.
+#
+# AuthservIDWithJobID false
+
+##  AutoRestart { true | false }
+##  	default "false"
+##
+##  Automatically re-start on failures. Use with caution; if the filter fails
+##  instantly after it starts, this can cause a tight fork(2) loop.
+#
+# AutoRestart false
+
+##  AutoRestartCount n
+##  	default 0
+##
+##  Sets the maximum automatic restart count.  After this number of automatic
+##  restarts, the filter will give up and terminate.  A value of 0 implies no
+##  limit.
+#
+# AutoRestartCount 0
+
+##  AutoRestartRate n/t[u]
+##  	default (no limit)
+##
+##  Sets the maximum automatic restart rate.  If the filter begins restarting
+##  faster than the rate defined here, it will give up and terminate.  This
+##  is a string of the form n/t[u] where n is an integer limiting the count
+##  of restarts in the given interval and t[u] defines the time interval
+##  through which the rate is calculated; t is an integer and u defines the
+##  units thus represented ("s" or "S" for seconds, the default; "m" or "M"
+##  for minutes; "h" or "H" for hours; "d" or "D" for days). For example, a
+##  value of "10/1h" limits the restarts to 10 in one hour. There is no
+##  default, meaning restart rate is not limited.
+#
+# AutoRestartRate n/t[u]
+
+##  Background { true | false }
+##  	default "true"
+##
+##  Causes opendmarc to fork and exits immediately, leaving the service
+##  running in the background.
+#
+# Background true
+
+##  BaseDirectory (string)
+##  	default (none)
+##
+##  If set, instructs the filter to change to the specified directory using
+##  chdir(2) before doing anything else.  This means any files referenced
+##  elsewhere in the configuration file can be specified relative to this
+##  directory.  It's also useful for arranging that any crash dumps will be
+##  saved to a specific location.
+#
+# BaseDirectory /var/run/opendmarc
+
+##  ChangeRootDirectory (string)
+##  	default (none)
+##
+##  Requests that the operating system change the effective root directory of
+##  the process to the one specified here prior to beginning execution.
+##  chroot(2) requires superuser access.  A warning will be generated if
+##  UserID is not also set.
+# 
+# ChangeRootDirectory /var/chroot/opendmarc
+
+##  CopyFailuresTo (string)
+##  	default (none)
+##
+##  Requests addition of the specified email address to the envelope of
+##  any message that fails the DMARC evaluation.
+#
+# CopyFailuresTo postmaster@localhost
+
+##  DNSTimeout (integer)
+##  	default 5
+## 
+##  Sets the DNS timeout in seconds.  A value of 0 causes an infinite wait.
+##  (NOT YET IMPLEMENTED)
 #
-AuthservID {{ mail_server_hostname }}
+# DNSTimeout 5
 
-##  ForensicReports { true | false }
-##      default "false"
+##  EnableCoredumps { true | false }
+##  	default "false"
 ##
-# ForensicReports false
+##  On systems that have such support, make an explicit request to the kernel
+##  to dump cores when the filter crashes for some reason.  Some modern UNIX
+##  systems suppress core dumps during crashes for security reasons if the
+##  user ID has changed during the lifetime of the process.  Currently only
+##  supported on Linux.
+#
+# EnableCoreDumps false
+
+##  FailureReports { true | false }
+##  	default "false"
+##
+##  Enables generation of failure reports when the DMARC test fails and the
+##  purported sender of the message has requested such reports.  Reports are
+##  formatted per RFC6591.
+# 
+# FailureReports false
 
-PidFile /var/run/opendmarc.pid
+##  FailureReportsBcc (string)
+##  	default (none)
+##
+##  When failure reports are enabled and one is to be generated, always
+##  send one to the address(es) specified here.  If a failure report is
+##  requested by the domain owner, the address(es) are added in a Bcc: field.
+##  If no request is made, they address(es) are used in a To: field.  There
+##  is no default.
+# 
+# FailureReportsBcc postmaster@example.coom
+
+##  FailureReportsOnNone { true | false }
+##  	default "false"
+##
+##  Supplements the "FailureReports" setting by generating reports for
+##  domains that advertise "none" policies.  By default, reports are only
+##  generated (when enabled) for sending domains advertising a "quarantine"
+##  or "reject" policy.
+# 
+# FailureReportsOnNone false
+
+##  FailureReportsSentBy string
+##  	default "USER@HOSTNAME"
+##
+##  Specifies the email address to use in the From: field of failure
+##  reports generated by the filter.  The default is to use the userid of
+##  the user running the filter and the local hostname to construct an
+##  email address.  "postmaster" is used in place of the userid if a name
+##  could not be determined.
+# 
+# FailureReportsSentBy USER@HOSTNAME
+
+##  HistoryFile path
+##  	default (none)
+##
+##  If set, specifies the location of a text file to which records are written
+##  that can be used to generate DMARC aggregate reports.  Records are groups
+##  of rows containing information about a single received message, and
+##  include all relevant information needed to generate a DMARC aggregate
+##  report.  It is expected that this will not be used in its raw form, but
+##  rather periodically imported into a relational database from which the
+##  aggregate reports can be extracted by a tool such as opendmarc-import(8).
+#
+# HistoryFile /var/run/opendmarc.dat
+
+##  IgnoreAuthenticatedClients { true | false }
+##  	default "false"
+##
+##  If set, causes mail from authenticated clients (i.e., those that used
+##  SMTP AUTH) to be ignored by the filter.
+#
+# IgnoreAuthenticatedClients false
+
+##  IgnoreHosts path
+##  	default (internal)
+##
+##  Specifies the path to a file that contains a list of hostnames, IP
+##  addresses, and/or CIDR expressions identifying hosts whose SMTP
+##  connections are to be ignored by the filter.  If not specified, defaults
+##  to "127.0.0.1" only.
+#
+# IgnoreHosts /usr/local/etc/opendmarc/ignore.hosts
+
+##  IgnoreMailFrom domain[,...]
+##  	default (none)
+##
+##  Gives a list of domain names whose mail (based on the From: domain) is to
+##  be ignored by the filter.  The list should be comma-separated.  Matching
+##  against this list is case-insensitive.  The default is an empty list,
+##  meaning no mail is ignored.
+#
+# IgnoreMailFrom example.com
+
+##  MilterDebug (integer)
+##  	default 0
+##
+##  Sets the debug level to be requested from the milter library.
+#
+# MilterDebug 0
+
+##  PidFile path
+##  	default (none)
+##
+##  Specifies the path to a file that should be created at process start
+##  containing the process ID.
+##
+#
+# PidFile /var/run/opendmarc.pid
+
+##  PublicSuffixList path
+##  	default (none)
+##
+##  Specifies the path to a file that contains top-level domains (TLDs) that
+##  will be used to compute the Organizational Domain for a given domain name,
+##  as described in the DMARC specification.  If not provided, the filter will
+##  not be able to determine the Organizational Domain and only the presented
+##  domain will be evaluated.
+#
+# PublicSuffixList path
+
+##  RecordAllMessages { true | false }
+##  	default "false"
+##
+##  If set and "HistoryFile" is in use, all received messages are recorded
+##  to the history file.  If not set (the default), only messages for which
+##  the From: domain published a DMARC record will be recorded in the
+##  history file.
+#
+# RecordAllMessages false
 
 ##  RejectFailures { true | false }
-##      default "false"
+##  	default "false"
 ##
-RejectFailures false
+##  If set, messages will be rejected if they fail the DMARC evaluation, or
+##  temp-failed if evaluation could not be completed.  By default, no message
+##  will be rejected or temp-failed regardless of the outcome of the DMARC
+##  evaluation of the message.  Instead, an Authentication-Results header
+##  field will be added.
+#
+# RejectFailures false
+
+##  ReportCommand string
+##  	default "/usr/sbin/sendmail -t"
+##
+##  Indicates the shell command to which failure reports should be passed for
+##  delivery when "FailureReports" is enabled.
+#
+# ReportCommand /usr/sbin/sendmail -t
+
+##  RequiredHeaders { true | false }
+##  	default "false"
+##
+##  If set, the filter will ensure the header of the message conforms to the
+##  basic header field count restrictions laid out in RFC5322, Section 3.6.
+##  Messages failing this test are rejected without further processing.  A
+##  From: field from which no domain name could be extracted will also be
+##  rejected.
+#
+# RequiredHeaders false
+
+##  Socket socketspec
+##  	default (none)
+##
+##  Specifies the socket that should be established by the filter to receive
+##  connections from sendmail(8) in order to provide service.  socketspec is
+##  in one of two forms: local:path, which creates a UNIX domain socket at
+##  the specified path, or inet:port[@host] or inet6:port[@host] which creates
+##  a TCP socket on the specified port for the appropriate protocol family.
+##  If the host is not given as either a hostname or an IP address, the
+##  socket will be listening on all interfaces.  This option is mandatory
+##  either in the configuration file or on the command line.  If an IP
+##  address is used, it must be enclosed in square brackets.
+#
+# Socket inet:8893@localhost
+
+##  SoftwareHeader { true | false }
+##  	default "false"
+##
+##  Causes the filter to add a "DMARC-Filter" header field indicating the
+##  presence of this filter in the path of the message from injection to
+##  delivery.  The product's name, version, and the job ID are included in
+##  the header field's contents.
+#
+# SoftwareHeader false
+
+##  SPFIgnoreResults { true | false }
+##	default "false"
+##
+##  Causes the filter to ignore any SPF results in the header of the
+##  message.  This is useful if you want the filter to perfrom SPF checks
+##  itself, or because you don't trust the arriving header.
+#
+# SPFIgnoreResults false
+
+##  SPFSelfValidate { true | false }
+##	default false
+##
+##  Enable internal spf checking with --with-spf
+##  To use libspf2 instead:  --with-spf --with-spf2-include=path --with-spf2-lib=path
+##
+##  Causes the filter to perform a fallback SPF check itself when
+##  it can find no SPF results in the message header.  If SPFIgnoreResults
+##  is also set, it never looks for SPF results in headers and
+##  always performs the SPF check itself when this is set.
+#
+# SPFSelfValidate false
 
 ##  Syslog { true | false }
-##      default "false"
+##  	default "false"
 ##
 ##  Log via calls to syslog(3) any interesting activity.
 #
-Syslog true
+# Syslog false
 
 ##  SyslogFacility facility-name
-##      default "mail"
+##  	default "mail"
 ##
 ##  Log via calls to syslog(3) using the named facility.  The facility names
 ##  are the same as the ones allowed in syslog.conf(5).
 #
 # SyslogFacility mail
 
+##  TemporaryDirectory path
+##  	default /var/tmp
+##
+##  Specifies the directory in which temporary files should be written.
+#
+# TemporaryDirectory /var/tmp
+
 ##  TrustedAuthservIDs string
-##      default HOSTNAME
+##  	default HOSTNAME
 ##
 ##  Specifies one or more "authserv-id" values to trust as relaying true
 ##  upstream DKIM and SPF results.  The default is to use the name of
@@ -43,11 +338,11 @@ Syslog true
 ##  with a comma.  The key word "HOSTNAME" will be replaced by the name of
 ##  the host running the filter as reported by the gethostname(3) function.
 #
-TrustedAuthservIDs {{ mail_server_hostname }}
+# TrustedAuthservIDs HOSTNAME
 
 
 ##  UMask mask
-##      default (none)
+##  	default (none)
 ##
 ##  Requests a specific permissions mask to be used for file creation.  This
 ##  only really applies to creation of the socket when Socket specifies a
@@ -56,30 +351,13 @@ TrustedAuthservIDs {{ mail_server_hostname }}
 ##  specific file mode on creation regardless of the process umask.  See
 ##  umask(2) for more information.
 #
-UMask 0002
+# UMask 077
 
 ##  UserID user[:group]
-##      default (none)
+##  	default (none)
 ##
 ##  Attempts to become the specified userid before starting operations.
 ##  The process will be assigned all of the groups and primary group ID of
 ##  the named userid unless an alternate group is specified.
 #
-UserID opendmarc:opendmarc
-
-## The path to the Ignored Hosts list. This file should contain a list of
-## networks and hosts that you trust. Their mail will not be checked by
-## OpenDMARC.
-#
-IgnoreHosts /etc/opendmarc/ignore.hosts
-
-## The path under which the History file should be created.
-## This file is necessary if you want to be able to create aggregate
-## reports to send out to other organizations
-#
-HistoryFile /var/run/opendmarc/opendmarc.dat
-
-## Adds a “Dmarc-Filter” header with the opendmarc version in every processed mail.
-## This is good to have during testing.
-#
-SoftwareHeader true
+# UserID opendmarc