zabbix_export: version: '7.2' media_types: - name: GitHub type: WEBHOOK parameters: - name: alert_message value: '{ALERT.MESSAGE}' - name: alert_subject value: '{ALERT.SUBJECT}' - name: event_id value: '{EVENT.ID}' - name: event_nseverity value: '{EVENT.NSEVERITY}' - name: event_severity value: '{EVENT.SEVERITY}' - name: event_source value: '{EVENT.SOURCE}' - name: event_update_nseverity value: '{EVENT.UPDATE.NSEVERITY}' - name: event_update_severity value: '{EVENT.UPDATE.SEVERITY}' - name: event_update_status value: '{EVENT.UPDATE.STATUS}' - name: event_value value: '{EVENT.VALUE}' - name: github_api_version value: '2022-11-28' - name: github_issue_number value: '{EVENT.TAGS.__zbx_github_issue_number}' - name: github_repo value: '{ALERT.SENDTO}' - name: github_token value: '' - name: github_url value: 'https://api.github.com' - name: github_user_agent value: Zabbix/7.2 - name: github_zabbix_event_priority_label_prefix value: 'Zabbix Event Priority: ' - name: github_zabbix_event_source_label_prefix value: 'Zabbix Event Source: ' - name: github_zabbix_event_status_label_prefix value: 'Zabbix Event Status: ' - name: github_zabbix_generic_label value: 'Zabbix GitHub Webhook' - name: trigger_id value: '{TRIGGER.ID}' - name: zabbix_url value: '{$ZABBIX.URL}' status: DISABLED script: | const CLogger = function(serviceName) { this.serviceName = serviceName; this.INFO = 4 this.WARN = 3 this.ERROR = 2 this.log = function(level, msg) { Zabbix.log(level, '[' + this.serviceName + '] ' + msg); } } const CWebhook = function(value) { try { params = JSON.parse(value); if (['0', '1', '2', '3', '4'].indexOf(params.event_source) === -1) { throw 'Incorrect "event_source" parameter given: ' + params.event_source + '.\nMust be 0-4.'; } if (['0', '3', '4'].indexOf(params.event_source) !== -1 && ['0', '1'].indexOf(params.event_value) === -1) { throw 'Incorrect "event_value" parameter given: ' + params.event_value + '.\nMust be 0 or 1.'; } if (['0', '3', '4'].indexOf(params.event_source) !== -1) { if (params.event_source === '1' && ['0', '1', '2', '3'].indexOf(params.event_value) === -1) { throw 'Incorrect "event_value" parameter given: ' + params.event_value + '.\nMust be 0-3.'; } if (params.event_source === '0' && ['0', '1'].indexOf(params.event_update_status) === -1) { throw 'Incorrect "event_update_status" parameter given: ' + params.event_update_status + '.\nMust be 0 or 1.'; } if (params.event_source === '4') { if (['0', '1', '2', '3', '4', '5'].indexOf(params.event_update_nseverity) !== -1 && params.event_update_nseverity != params.event_nseverity) { params.event_nseverity = params.event_update_nseverity; params.event_severity = params.event_update_severity; params.event_update_status = '1'; } } } this.runCallback = function(name, params) { if (typeof this[name] === 'function') { return this[name].apply(this, [params]); } } this.handleEvent = function(source, event) { const alert = { source: source, event: event }; return [ this.runCallback('on' + source + event, alert), this.runCallback('on' + event, alert), this.runCallback('onEvent', alert) ]; } this.handleEventless = function(source) { const alert = { source: source, event: null }; return [ this.runCallback('on' + source, alert), this.runCallback('onEvent', alert) ]; } this.run = function() { var results = []; if (typeof this.httpProxy === 'string' && this.httpProxy.trim() !== '') { this.request.setProxy(this.httpProxy); } const types = { '0': 'Trigger', '1': 'Discovery', '2': 'Autoreg', '3': 'Internal', '4': 'Service' }; if (['0', '3', '4'].indexOf(this.params.event_source) !== -1) { var event = (this.params.event_update_status === '1') ? 'Update' : ((this.params.event_value === '1') ? 'Problem' : 'Resolve'); results = this.handleEvent(types[this.params.event_source], event); } else if (typeof types[this.params.event_source] !== 'undefined') { results = this.handleEventless(types[this.params.event_source]); } else { throw 'Unexpected "event_source": ' + this.params.event_source; } for (idx in results) { if (typeof results[idx] !== 'undefined') { return JSON.stringify(results[idx]); } } } this.httpProxy = params.http_proxy; this.params = params; this.runCallback('onCheckParams', {}); } catch (error) { throw 'Webhook processing failed: ' + error; } } const CParamValidator = { isType: function(value, type) { if (type === 'array') { return Array.isArray(value); } if (type === 'integer') { return CParamValidator.isInteger(value); } if (type === 'float') { return CParamValidator.isFloat(value); } return (typeof value === type); }, isInteger: function(value) { if (!CParamValidator.ifMatch(value, /^-?\d+$/)) { return false; } return !isNaN(parseInt(value)); }, isFloat: function(value) { if (!CParamValidator.ifMatch(value, /^-?\d+\.\d+$/)) { return false; } return !isNaN(parseFloat(value)); }, isDefined: function(value) { return !CParamValidator.isType(value, 'undefined'); }, isEmpty: function(value) { if (!CParamValidator.isType(value, 'string')) { throw 'Value "' + value + '" must be a string to be checked for emptiness.'; } return (value.trim() === ''); }, isMacroSet: function(value, macro) { if (CParamValidator.isDefined(macro)) { return !(CParamValidator.ifMatch(value, '^\{' + macro + '\}$')) } return !(CParamValidator.ifMatch(value, '^\{[$#]{0,1}[A-Z_\.]+[\:]{0,1}["]{0,1}.*["]{0,1}\}$') || value === '*UNKNOWN*') }, withinRange: function(value, min, max) { if (!CParamValidator.isType(value, 'number')) { throw 'Value "' + value + '" must be a number to be checked for range.'; } if (value < ((CParamValidator.isDefined(min)) ? min : value) || value > ((CParamValidator.isDefined(max)) ? max : value)) { return false; } return true; }, inArray: function(value, array) { if (!CParamValidator.isType(array, 'array')) { throw 'The array must be an array to check the value for existing in it.'; } return (array.indexOf((typeof value === 'string') ? value.toLowerCase() : value) !== -1); }, ifMatch: function(value, regex) { return (new RegExp(regex)).test(value); }, match: function(value, regex) { if (!CParamValidator.isType(value, 'string')) { throw 'Value "' + value + '" must be a string to be matched with the regular expression.'; } return value.match(new RegExp(regex)); }, checkURL: function(value) { if (CParamValidator.isEmpty(value)) { throw 'URL value "' + value + '" must be a non-empty string.'; } if (!CParamValidator.ifMatch(value, '^(http|https):\/\/.+')) { throw 'URL value "' + value + '" must contain a schema.'; } return value.endsWith('/') ? value.slice(0, -1) : value; }, check: function(key, rule, params) { if (!CParamValidator.isDefined(rule.type)) { throw 'Mandatory attribute "type" has not been defined for parameter "' + key + '".'; } if (!CParamValidator.isDefined(params[key])) { throw 'Checked parameter "' + key + '" was not found in the list of input parameters.'; } var value = params[key], error_message = null; switch (rule.type) { case 'string': if (!CParamValidator.isType(value, 'string')) { throw 'Value "' + key + '" must be a string.'; } if (CParamValidator.isEmpty(value)) { error_message = 'Value "' + key + '" must be a non-empty string'; break; } if (CParamValidator.isDefined(rule.len) && value.length < rule.len) { error_message = 'Value "' + key + '" must be a string with a length > ' + rule.len; } if (CParamValidator.isDefined(rule.regex) && !CParamValidator.ifMatch(value, rule.regex)) { error_message = 'Value "' + key + '" must match the regular expression "' + rule.regex + '"'; } if (CParamValidator.isDefined(rule.url) && rule.url === true) { value = CParamValidator.checkURL(value); } break; case 'integer': if (!CParamValidator.isInteger(value)) { error_message = 'Value "' + key + '" must be an integer'; break; } value = parseInt(value); break; case 'float': if (!CParamValidator.isFloat(value)) { error_message = 'Value "' + key + '" must be a floating-point number'; break; } value = parseFloat(value); break; case 'boolean': if (CParamValidator.inArray(value, ['1', 'true', 'yes', 'on'])) { value = true; } else if (CParamValidator.inArray(value, ['0', 'false', 'no', 'off'])) { value = false; } else { error_message = 'Value "' + key + '" must be a boolean-like.'; } break; case 'array': try { value = JSON.parse(value); } catch (error) { throw 'Value "' + key + '" contains invalid JSON.'; } if (!CParamValidator.isType(value, 'array')) { error_message = 'Value "' + key + '" must be an array.'; } if (CParamValidator.isDefined(rule.tags) && rule.tags === true) { value = value.reduce(function(acc, obj) { acc[obj.tag] = obj.value || null; return acc; }, {}); } break; case 'object': value = JSON.parse(value); if (!CParamValidator.isType(value, 'object')) { error_message = 'Value "' + key + '" must be an object.'; } break; default: throw 'Unexpected attribute type "' + rule.type + '" for value "' + key + '". Available: ' + ['integer', 'float', 'string', 'boolean', 'array', 'object'].join(', '); } params[key] = value; if (CParamValidator.inArray(rule.type, ['integer', 'float']) && error_message === null && (CParamValidator.isDefined(rule.min) || CParamValidator.isDefined(rule.max)) && !CParamValidator.withinRange(value, rule.min, rule.max)) { error_message = 'Value "' + key + '" must be a number ' + ((CParamValidator.isDefined(rule.min) && CParamValidator.isDefined(rule.max)) ? (rule.min + '..' + rule.max) : ((CParamValidator.isDefined(rule.min)) ? '>' + rule.min : '<' + rule.max)); } else if (CParamValidator.isDefined(rule.array) && !CParamValidator.inArray(value, rule.array)) { error_message = 'Value "' + key + '" must be in the array ' + JSON.stringify(rule.array); } else if (CParamValidator.isDefined(rule.macro) && !CParamValidator.isMacroSet(value.toString(), rule.macro)) { error_message = 'The macro ' + ((CParamValidator.isDefined(rule.macro)) ? '{' + rule.macro + '} ' : ' ') + 'is not set'; } if (error_message !== null) { if (CParamValidator.isDefined(rule.default) && CParamValidator.isType(rule.default, rule.type)) { params[key] = rule.default; } else { Zabbix.log(4, 'Default value for "' + key + '" must be a ' + rule.type + '. Skipped.'); throw 'Incorrect value for variable "' + key + '". ' + error_message; } } return this; }, validate: function(rules, params) { if (!CParamValidator.isType(params, 'object') || CParamValidator.isType(params, 'array')) { throw 'Incorrect parameters value. The value must be an object.'; } for (var key in rules) { CParamValidator.check(key, rules[key], params); } } } const CHttpRequest = function(logger) { this.request = new HttpRequest(); if (typeof logger !== 'object' || logger === null) { this.logger = Zabbix; } else { this.logger = logger; } this.clearHeader = function() { this.request.clearHeader(); } this.addHeaders = function(value) { var headers = []; if (typeof value === 'object' && value !== null) { if (!Array.isArray(value)) { Object.keys(value).forEach(function(key) { headers.push(key + ': ' + value[key]); }); } else { headers = value; } } else if (typeof value === 'string') { value.split('\r\n').forEach(function(header) { headers.push(header); }); } for (var idx in headers) { this.request.addHeader(headers[idx]); } } this.setProxy = function(proxy) { this.request.setProxy(proxy); } this.plainRequest = function(method, url, data) { var resp = null; method = method.toLowerCase(); this.logger.log(4, 'Sending ' + method + ' request:' + JSON.stringify(data)); if (['get', 'post', 'put', 'patch', 'delete', 'trace'].indexOf(method) !== -1) { resp = this.request[method](url, data); } else if (['connect', 'head', 'options'].indexOf(method) !== -1) { resp = this.request[method](url); } else { throw 'Unexpected method. Method ' + method + ' is not supported.'; } this.logger.log(4, 'Response has been received: ' + resp); return resp; } this.jsonRequest = function(method, url, data) { this.addHeaders('Content-Type: application/json'); var resp = this.plainRequest(method, url, JSON.stringify(data)); try { resp = JSON.parse(resp); } catch (error) { throw 'Failed to parse response: not well-formed JSON was received'; } return resp; } this.getStatus = function() { return this.request.getStatus(); } } const CWebhookHelper = { createProblemURL: function(event_source, zabbix_url, trigger_id, event_id) { if (event_source === '0') { return zabbix_url + '/tr_events.php?triggerid=' + trigger_id + '&eventid=' + event_id; } else if (event_source === '4') { return zabbix_url + '/zabbix.php?action=service.list'; } return zabbix_url; }, }; var serviceLogName = 'GitHub Webhook', Logger = new CLogger(serviceLogName), GitHub = CWebhook; GitHub.prototype.onCheckParams = function () { CParamValidator.validate({ alert_message: {type: 'string'}, alert_subject: {type: 'string'}, github_api_version: {type: 'string'}, github_repo: {type: 'string'}, github_token: {type: 'string'}, github_url: {type: 'string', url: true}, github_user_agent: {type: 'string'}, github_zabbix_event_priority_label_prefix: {type: 'string', default: 'Zabbix Event Priority: '}, github_zabbix_event_source_label_prefix: {type: 'string', default: 'Zabbix Event Source: '}, github_zabbix_event_status_label_prefix: {type: 'string', default: 'Zabbix Event Status: '}, github_zabbix_generic_label: {type: 'string', default: 'Zabbix GitHub Webhook'}, zabbix_url: {type: 'string', url: true} }, this.params); this.request_headers = { 'User-Agent': this.params.github_user_agent, 'Accept': 'application/vnd.github+json', 'X-GitHub-Api-Version': this.params.github_api_version, 'Authorization': 'Bearer ' + this.params.github_token }; this.payload_data = { title: this.params.alert_subject, labels: [ { name: this.params.github_zabbix_generic_label } ] }; this.result = {tags: {}}; }; function checkResponse(response, received_code, required_code, response_field, error_message) { if (received_code != required_code || !CParamValidator.isDefined(response[response_field])) { var message = error_message + ' Request failed with status code ' + received_code; if (CParamValidator.isDefined(response.message) && Object.keys(response.message).length > 0) { message += ': ' + response.message; } throw message + ' Check debug log for more information.'; } } GitHub.prototype.createIssue = function () { this.payload_data.body = this.params.alert_message + '\n' + CWebhookHelper.createProblemURL(this.params.event_source, this.params.zabbix_url, this.params.trigger_id, this.params.event_id); this.request.addHeaders(this.request_headers); const response = this.request.jsonRequest('POST', this.params.github_url + '/repos/' + this.params.github_repo + '/issues', this.payload_data); checkResponse(response, this.request.getStatus(), 201, 'number', 'Cannot create GitHub issue.'); return response; } GitHub.prototype.updateIssue = function () { this.request.addHeaders(this.request_headers); const response = this.request.jsonRequest('PATCH', this.params.github_url + '/repos/' + this.params.github_repo + '/issues/' + this.params.github_issue_number, this.payload_data); checkResponse(response, this.request.getStatus(), 200, 'number', 'Cannot update GitHub issue.'); return response; } GitHub.prototype.addIssueComment = function () { this.payload_data = { body: this.params.alert_message }; this.request.addHeaders(this.request_headers); const response = this.request.jsonRequest('POST', this.params.github_url + '/repos/' + this.params.github_repo + '/issues/' + this.params.github_issue_number + '/comments', this.payload_data); checkResponse(response, this.request.getStatus(), 201, 'id', 'Cannot add comment for GitHub issue.'); return response; } GitHub.prototype.onProblem = function (alert) { if (CParamValidator.isMacroSet(this.params.github_issue_number)) { return this.onUpdate(alert); } Logger.log(Logger.INFO, 'Source: ' + alert.source + '; Event: ' + alert.event); if (this.params.event_source === '0') { CParamValidator.validate({event_id: {type: 'integer'}, trigger_id: {type: 'integer'}}, this.params); } this.payload_data.labels.push({name: this.params.github_zabbix_event_source_label_prefix + alert.source}); this.payload_data.labels.push({name: this.params.github_zabbix_event_status_label_prefix + 'Problem'}); if (!CParamValidator.isEmpty(this.params.event_severity) && CParamValidator.isMacroSet(this.params.event_severity, 'EVENT.SEVERITY')) { this.payload_data.labels.push({name: this.params.github_zabbix_event_priority_label_prefix + this.params.event_severity}); } const response = this.createIssue(); this.result.tags = { __zbx_github_issue_number: response.number, __zbx_github_repo: this.params.github_repo, __zbx_github_link: response.html_url }; return this.result; } GitHub.prototype.onUpdate = function (alert) { Logger.log(Logger.INFO, 'Source: ' + alert.source + '; Event: ' + alert.event); if (!CParamValidator.isMacroSet(this.params.github_issue_number)) { throw "Failed to update the existing issue: no issue number was received." } this.payload_data.labels.push({name: this.params.github_zabbix_event_source_label_prefix + alert.source}); if (this.params.event_value === '0') { this.payload_data.labels.push({name: this.params.github_zabbix_event_status_label_prefix + 'Resolved'}); } else { this.payload_data.labels.push({name: this.params.github_zabbix_event_status_label_prefix + 'Problem'}); } if (!CParamValidator.isEmpty(this.params.event_severity) && CParamValidator.isMacroSet(this.params.event_severity, 'EVENT.SEVERITY')) { this.payload_data.labels.push({name: this.params.github_zabbix_event_priority_label_prefix + this.params.event_severity}); } this.updateIssue(); this.request.clearHeader(); this.addIssueComment(); return this.result; } GitHub.prototype.onResolve = function (alert) { return this.onUpdate(alert); } GitHub.prototype.onDiscovery = function (alert) { Logger.log(Logger.INFO, 'Source: ' + alert.source + '; Event: ' + alert.event); this.payload_data.labels.push({name: this.params.github_zabbix_event_source_label_prefix + alert.source}); this.createIssue(); return this.result; } GitHub.prototype.onAutoreg = function (alert) { return this.onDiscovery(alert); } try { var hook = new GitHub(value); hook.request = new CHttpRequest(Logger); return hook.run(); } catch (error) { Logger.log(Logger.WARN, 'notification failed: ' + error); throw 'Sending failed: ' + error; } process_tags: 'YES' show_event_menu: 'YES' event_menu_url: '{EVENT.TAGS.__zbx_github_link}' event_menu_name: 'Github: Issue {EVENT.TAGS.__zbx_github_issue_number}' description: | This media type integrates your Zabbix installation with GitHub using the Zabbix webhook feature. GitHub configuration: 1. Create an access token. One of the simplest ways to send authenticated requests is to use a personal access token - either a classic or a fine-grained one: https://docs.github.com/en/rest/authentication/authenticating-to-the-rest-api?apiVersion=2022-11-28#authenticating-with-a-personal-access-token Classic personal access token You can create a new classic personal access token by following the instructions in the official documentation: https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#creating-a-personal-access-token-classic The token user must have a permission to create issues and issue comments in the desired repositories. For webhook to work on private repositories, the "repo" scope must be set in token settings to have full control of private repositories. Additional information about OAuth scopes is available in the official documentation: https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/scopes-for-oauth-apps#available-scopes Fine-grained personal access token Alternatively, you can use a fine-grained personal access token: https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#creating-a-fine-grained-personal-access-token In order to use fine-grained tokens to monitor organization-owned repositories, organizations must opt in to fine-grained personal access tokens and set up a personal access token policy: https://docs.github.com/en/organizations/managing-programmatic-access-to-your-organization/setting-a-personal-access-token-policy-for-your-organization The fine-grained token needs to have the following permission set to provide access to the repository issues: - "Issues" repository permissions (write) 2. Copy and save the created token somewhere, as it will be shown only once for security reasons. Zabbix configuration: 1. Before you can start using Zammad webhook, set up the global macro "{$ZABBIX.URL}": - In the Zabbix web interface, go to "Administration" → "Macros" section in the dropdown menu in the top left corner. - Set up the global macro "{$ZABBIX.URL}" which will contain the URL to the Zabbix frontend. The URL should be either an IP address, a fully qualified domain name, or localhost. - Specifying a protocol is mandatory, whereas the port is optional. Depending on the web server configuration you might also need to append "/zabbix" to the end of URL. Good examples: - http://zabbix.com - https://zabbix.lan/zabbix - http://server.zabbix.lan/ - http://localhost - http://127.0.0.1:8080 - Bad examples: - zabbix.com - http://zabbix/ 2. Set the "github_token" webhook parameter value to the access token that you created previously. You can also adjust the issue labels created by the webhook in the following parameters: - github_zabbix_event_priority_label_prefix - the prefix for the issue label that displays the Zabbix event priority in the supported event sources. It is set to "Zabbix Event Priority: " by default. - github_zabbix_event_source_label_prefix - the prefix for the issue label that displays the Zabbix event source. It is set to "Zabbix Event Source: " by default. - github_zabbix_event_status_label_prefix - the prefix for the issue label that displays the Zabbix event status. It is set to "Zabbix Event Status: " by default. - github_zabbix_generic_label - the label that is added to all issues created by the webhook. It is set to "Zabbix GitHub Webhook" by default. Note that the webhook will reuse the labels with the same name that already exist in the repository (including the color, so it can changed from the default value for new labels in GitHub, if needed). Also, the labels are replaced when the issue is updated, so any user-added labels will be removed. 4. Create a Zabbix user and add media: - If you want to create a new user, go to the "Users" → "Users" section, click the "Create user" button in the top right corner. In the "User" tab, fill in all required fields (marked with red asterisks). - In the "Media" tab, add a new media and select "Zammad" type from the drop-down list. In field "Send to" specify the full repo name (owner/project name) e.g. johndoe/example-project. - Make sure this user has access to all hosts for which you would like problem notifications to be sent to GitHub. 5. Great! You can now start using this media type in actions and create GitHub issues! You can find the latest version of this media and additional information in the official Zabbix repository: https://git.zabbix.com/projects/ZBX/repos/zabbix/browse/templates/media/github message_templates: - event_source: TRIGGERS operation_mode: PROBLEM subject: 'Problem: {EVENT.NAME}' message: | Problem started at {EVENT.TIME} on {EVENT.DATE} Problem name: {EVENT.NAME} Host: {HOST.NAME} Severity: {EVENT.SEVERITY} Operational data: {EVENT.OPDATA} Original problem ID: {EVENT.ID} {TRIGGER.URL} - event_source: TRIGGERS operation_mode: RECOVERY subject: 'Resolved in {EVENT.DURATION}: {EVENT.NAME}' message: | Problem has been resolved in {EVENT.DURATION} at {EVENT.RECOVERY.TIME} on {EVENT.RECOVERY.DATE} Problem name: {EVENT.NAME} Host: {HOST.NAME} Severity: {EVENT.SEVERITY} Original problem ID: {EVENT.ID} {TRIGGER.URL} - event_source: TRIGGERS operation_mode: UPDATE subject: 'Updated problem in {EVENT.AGE}: {EVENT.NAME}' message: | {USER.FULLNAME} {EVENT.UPDATE.ACTION} problem at {EVENT.UPDATE.DATE} {EVENT.UPDATE.TIME}. {EVENT.UPDATE.MESSAGE} Current problem status is {EVENT.STATUS}, age is {EVENT.AGE}, acknowledged: {EVENT.ACK.STATUS}. - event_source: DISCOVERY operation_mode: PROBLEM subject: 'Discovery: {DISCOVERY.DEVICE.STATUS} {DISCOVERY.DEVICE.IPADDRESS}' message: | Discovery rule: {DISCOVERY.RULE.NAME} Device IP: {DISCOVERY.DEVICE.IPADDRESS} Device DNS: {DISCOVERY.DEVICE.DNS} Device status: {DISCOVERY.DEVICE.STATUS} Device uptime: {DISCOVERY.DEVICE.UPTIME} Device service name: {DISCOVERY.SERVICE.NAME} Device service port: {DISCOVERY.SERVICE.PORT} Device service status: {DISCOVERY.SERVICE.STATUS} Device service uptime: {DISCOVERY.SERVICE.UPTIME} - event_source: AUTOREGISTRATION operation_mode: PROBLEM subject: 'Autoregistration: {HOST.HOST}' message: | Host name: {HOST.HOST} Host IP: {HOST.IP} Agent port: {HOST.PORT} - event_source: INTERNAL operation_mode: PROBLEM subject: '[{EVENT.STATUS}] {EVENT.NAME}' message: | Problem started at {EVENT.TIME} on {EVENT.DATE} Problem name: {EVENT.NAME} Host: {HOST.NAME} Original problem ID: {EVENT.ID} - event_source: INTERNAL operation_mode: RECOVERY subject: '[{EVENT.STATUS}] {EVENT.NAME}' message: | Problem has been resolved in {EVENT.DURATION} at {EVENT.RECOVERY.TIME} on {EVENT.RECOVERY.DATE} Problem name: {EVENT.NAME} Host: {HOST.NAME} Original problem ID: {EVENT.ID} - event_source: SERVICE operation_mode: PROBLEM subject: 'Service "{SERVICE.NAME}" problem: {EVENT.NAME}' message: | Service problem started at {EVENT.TIME} on {EVENT.DATE} Service problem name: {EVENT.NAME} Service: {SERVICE.NAME} Severity: {EVENT.SEVERITY} Original problem ID: {EVENT.ID} Service description: {SERVICE.DESCRIPTION} {SERVICE.ROOTCAUSE} - event_source: SERVICE operation_mode: RECOVERY subject: 'Service "{SERVICE.NAME}" resolved in {EVENT.DURATION}: {EVENT.NAME}' message: | Service "{SERVICE.NAME}" has been resolved at {EVENT.RECOVERY.TIME} on {EVENT.RECOVERY.DATE} Problem name: {EVENT.NAME} Problem duration: {EVENT.DURATION} Severity: {EVENT.SEVERITY} Original problem ID: {EVENT.ID} Service description: {SERVICE.DESCRIPTION} - event_source: SERVICE operation_mode: UPDATE subject: 'Changed "{SERVICE.NAME}" service status to {EVENT.UPDATE.SEVERITY} in {EVENT.AGE}' message: | Changed "{SERVICE.NAME}" service status to {EVENT.UPDATE.SEVERITY} at {EVENT.UPDATE.DATE} {EVENT.UPDATE.TIME}. Current problem age is {EVENT.AGE}. Service description: {SERVICE.DESCRIPTION} {SERVICE.ROOTCAUSE}