Add collectd to the monitoring suite

This change set builds collectd from source and configures it in one of
the following ways:

- If Librato credentials are present, collectd will be configured to
send data points to Librato using the collectd-librato plugin.

- If no Librato credentials are present, collectd will be configured to
write RRD files locally (/opt/collectd/var/lib/collectd/rrd by default).

README.textile

@@ -43,6 +43,7 @@ What do you get if you point this thing at a VPS? All kinds of good stuff!
 * Your own VPN server via "OpenVPN":http://openvpn.net/index.php/open-source.html.
 * An IRC bouncer via "ZNC":http://wiki.znc.in/ZNC.
 * "Monit":http://mmonit.com/monit/ to keep everything running smoothly (and alert you when it's not).
+* "collectd":http://collectd.org/ to collect system statistics.
 * Web hosting (ex: for your blog) via "Apache":https://www.apache.org/.
 * Firewall management via "Uncomplicated Firewall (ufw)":https://wiki.ubuntu.com/UncomplicatedFirewall.
 * Intrusion prevention via "fail2ban":http://www.fail2ban.org/ and rootkit detection via "rkhunter":http://rkhunter.sourceforge.net.

roles/monitoring/files/etc_init.d_collectd

@@ -0,0 +1,206 @@
+#! /bin/bash
+#
+# collectd - start and stop the statistics collection daemon
+# http://collectd.org/
+#
+# Copyright (C) 2005-2006 Florian Forster <octo@verplant.org>
+# Copyright (C) 2006-2009 Sebastian Harl <tokkee@debian.org>
+#
+
+### BEGIN INIT INFO
+# Provides:          collectd
+# Required-Start:    $local_fs $remote_fs
+# Required-Stop:     $local_fs $remote_fs
+# Should-Start:      $network $named $syslog $time cpufrequtils
+# Should-Stop:       $network $named $syslog
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: manage the statistics collection daemon
+# Description:       collectd is the statistics collection daemon.
+#                    It is a small daemon which collects system information
+#                    periodically and provides mechanisms to monitor and store
+#                    the values in a variety of ways.
+### END INIT INFO
+
+. /lib/lsb/init-functions
+
+export PATH=/opt/collectd/sbin:/opt/collectd/bin:/sbin:/bin:/usr/sbin:/usr/bin
+
+DISABLE=0
+
+DESC="statistics collection and monitoring daemon"
+NAME=collectd
+DAEMON=/opt/collectd/sbin/collectd
+
+CONFIGFILE=/opt/collectd/etc/collectd.conf
+PIDFILE=/opt/collectd/var/run/collectd.pid
+
+USE_COLLECTDMON=1
+COLLECTDMON_DAEMON=/opt/collectd/sbin/collectdmon
+COLLECTDMON_PIDFILE=/opt/collectd/var/run/collectdmon.pid
+
+MAXWAIT=30
+
+# Gracefully exit if the package has been removed.
+test -x $DAEMON || exit 0
+
+if [ -r /etc/default/$NAME ]; then
+	. /etc/default/$NAME
+fi
+
+if test "$ENABLE_COREFILES" == 1; then
+	ulimit -c unlimited
+fi
+
+if test "$USE_COLLECTDMON" == 1; then
+	_PIDFILE="$COLLECTDMON_PIDFILE"
+else
+	_PIDFILE="$PIDFILE"
+fi
+
+# return:
+#   0 if config is fine
+#   1 if there is a syntax error
+#   2 if there is no configuration
+check_config() {
+	if test ! -e "$CONFIGFILE"; then
+		return 2
+	fi
+	if ! $DAEMON -t -C "$CONFIGFILE"; then
+		return 1
+	fi
+	return 0
+}
+
+# return:
+#   0 if the daemon has been started
+#   1 if the daemon was already running
+#   2 if the daemon could not be started
+#   3 if the daemon was not supposed to be started
+d_start() {
+	if test "$DISABLE" != 0; then
+		# we get here during restart
+		log_progress_msg "disabled by /etc/default/$NAME"
+		return 3
+	fi
+
+	if test ! -e "$CONFIGFILE"; then
+		# we get here during restart
+		log_progress_msg "disabled, no configuration ($CONFIGFILE) found"
+		return 3
+	fi
+
+	check_config
+	rc="$?"
+	if test "$rc" -ne 0; then
+		log_progress_msg "not starting, configuration error"
+		return 2
+	fi
+
+	if test "$USE_COLLECTDMON" == 1; then
+		start-stop-daemon --start --quiet --oknodo --pidfile "$_PIDFILE" \
+			--exec $COLLECTDMON_DAEMON -- -P "$_PIDFILE" -- -C "$CONFIGFILE" \
+			|| return 2
+	else
+		start-stop-daemon --start --quiet --oknodo --pidfile "$_PIDFILE" \
+			--exec $DAEMON -- -C "$CONFIGFILE" -P "$_PIDFILE" \
+			|| return 2
+	fi
+	return 0
+}
+
+still_running_warning="
+WARNING: $NAME might still be running.
+In large setups it might take some time to write all pending data to
+the disk. You can adjust the waiting time in /etc/default/collectd."
+
+# return:
+#   0 if the daemon has been stopped
+#   1 if the daemon was already stopped
+#   2 if daemon could not be stopped
+d_stop() {
+	PID=$( cat "$_PIDFILE" 2> /dev/null ) || true
+
+	start-stop-daemon --stop --quiet --oknodo --pidfile "$_PIDFILE"
+	rc="$?"
+
+	if test "$rc" -eq 2; then
+		return 2
+	fi
+
+	sleep 1
+	if test -n "$PID" && kill -0 $PID 2> /dev/null; then
+		i=0
+		while kill -0 $PID 2> /dev/null; do
+			i=$(( $i + 2 ))
+			echo -n " ."
+
+			if test $i -gt $MAXWAIT; then
+				log_progress_msg "$still_running_warning"
+				return 2
+			fi
+
+			sleep 2
+		done
+		return "$rc"
+	fi
+	return "$rc"
+}
+
+case "$1" in
+	start)
+		log_daemon_msg "Starting $DESC" "$NAME"
+		d_start
+		case "$?" in
+			0|1) log_end_msg 0 ;;
+			2) log_end_msg 1 ;;
+			3) log_end_msg 255; true ;;
+			*) log_end_msg 1 ;;
+		esac
+		;;
+	stop)
+		log_daemon_msg "Stopping $DESC" "$NAME"
+		d_stop
+		case "$?" in
+			0|1) log_end_msg 0 ;;
+			2) log_end_msg 1 ;;
+		esac
+		;;
+	status)
+		status_of_proc -p "$_PIDFILE" "$DAEMON" "$NAME" && exit 0 || exit $?
+		;;
+	restart|force-reload)
+		log_daemon_msg "Restarting $DESC" "$NAME"
+		check_config
+		rc="$?"
+		if test "$rc" -eq 1; then
+			log_progress_msg "not restarting, configuration error"
+			log_end_msg 1
+			exit 1
+		fi
+		d_stop
+		rc="$?"
+		case "$rc" in
+			0|1)
+				sleep 1
+				d_start
+				rc2="$?"
+				case "$rc2" in
+					0|1) log_end_msg 0 ;;
+					2) log_end_msg 1 ;;
+					3) log_end_msg 255; true ;;
+					*) log_end_msg 1 ;;
+				esac
+				;;
+			*)
+				log_end_msg 1
+				;;
+		esac
+		;;
+	*)
+		echo "Usage: $0 {start|stop|restart|force-reload|status}" >&2
+		exit 3
+		;;
+esac
+
+# vim: syntax=sh noexpandtab sw=4 ts=4 :

roles/monitoring/handlers/main.yml

@@ -1,2 +1,5 @@
 - name: restart monit
-  service: name=monit state=restarted
+  service: name=monit state=restarted
+
+- name: restart collectd
+  service: name=collectd state=restarted

roles/monitoring/tasks/collectd.yml

@@ -0,0 +1,46 @@
+- name: Install collectd dependencies
+  apt: pkg={{ item }} state=installed
+  with_items:
+    - libcurl4-openssl-dev
+    - librrd2-dev
+    - python-dev
+
+- name: Download collectd
+  get_url: url=http://collectd.org/files/collectd-{{collectd_version}}.tar.gz
+           dest=/root/collectd-{{collectd_version}}.tar.gz
+
+- name: Extract collectd
+  command: tar xzf collectd-{{collectd_version}}.tar.gz
+           chdir=/root creates=/root/collectd-{{collectd_version}}
+
+- name: Build and install collectd
+  shell: ./configure ; make all ; make install
+         executable=/bin/bash
+         chdir=/root/collectd-{{collectd_version}}
+         creates=/opt/collectd/sbin/collectdmon
+
+- name: Copy collectd init file into place
+  copy: src=etc_init.d_collectd dest=/etc/init.d/collectd mode=0755
+
+- name: Download collectd-librato plugin
+  get_url: url=https://github.com/librato/collectd-librato/archive/v{{collectd_librato_version}}.tar.gz
+           dest=/root/collectd-librato-{{collectd_librato_version}}.tar.gz
+  when: collectd_librato_email|length > 0
+
+- name: Extract collectd-librato plugin
+  command: tar xzf collectd-librato-{{collectd_librato_version}}.tar.gz
+           chdir=/root creates=/root/collectd-librato-{{collectd_librato_version}}
+  when: collectd_librato_email|length > 0
+
+- name: Install collectd-librato plugin
+  command: make install
+           chdir=/root/collectd-librato-{{collectd_librato_version}}
+           creates=/opt/collectd-librato-{{collectd_librato_version}}
+  when: collectd_librato_email|length > 0
+
+- name: Copy collectd configuration file into place
+  template: src=opt_etc_collectd.conf.j2 dest=/opt/collectd/etc/collectd.conf
+  notify: restart collectd
+
+- name: Ensure collectd is a system service
+  service: name=collectd state=started enabled=true

roles/monitoring/tasks/main.yml

@@ -1,2 +1,3 @@
 - include: monit.yml tags=monit
 - include: logwatch.yml tags=logwatch
+- include: collectd.yml tags=collectd

roles/monitoring/templates/opt_etc_collectd.conf.j2

@@ -0,0 +1,32 @@
+BaseDir     "/opt/collectd"
+
+LoadPlugin syslog
+LoadPlugin cpu
+LoadPlugin interface
+LoadPlugin load
+LoadPlugin memory
+LoadPlugin disk
+LoadPlugin df
+
+{% if (collectd_librato_email|length and collectd_librato_api_token|length) %}
+<LoadPlugin python>
+  Globals true
+</LoadPlugin>
+
+<Plugin python>
+  ModulePath "/opt/collectd-librato-{{ collectd_librato_version }}/lib"
+  Import "collectd-librato"
+
+  <Module "collectd-librato">
+    Email    "{{ collectd_librato_email }}"
+    APIToken "{{ collectd_librato_api_token }}"
+    TypesDB  "/opt/collectd/share/collectd/types.db"
+  </Module>
+</Plugin>
+{% else %}
+LoadPlugin rrdtool
+
+<Plugin rrdtool>
+  DataDir "/opt/collectd/var/lib/collectd/rrd"
+</Plugin>
+{% endif %}

vars/defaults.yml

@@ -27,6 +27,12 @@ ntp_servers:
   # - 2.north-america.pool.ntp.org
   # - 3.north-america.pool.ntp.org
 
+# collectd
+collectd_version: 5.4.1
+collectd_librato_version: 0.0.10
+collectd_librato_email: "" # (optional)
+collectd_librato_api_token: "" # (optional)
+
 # google authenticator
 google_auth_version: 1.0