Add kibana deployment

kibana/Jenkinsfile

+#!/usr/bin/env groovy
+node {
+    common = load "/opt/jenkins/terraform/common.groovy"
+}
+
+pipeline {
+    agent any
+    parameters {
+        booleanParam(name: 'Recreate', defaultValue: false, description: 'Delete and create VM')
+        string(name: 'KibanaVersion', defaultValue: '7.9.0', description: 'Kibana version', trim: true)
+    }
+    environment {
+        RSYNC = 'rsync -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" --exclude ".git"'
+        SSH = 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
+        SCP = 'scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
+        TLD = 'riaint.ee'
+        NODE = "${env.JOB_BASE_NAME}-01.dev.${TLD}"
+        PROJECT_NAME = 'jut'
+        PROJECT_PATH = "${PROJECT_NAME}/${env.JOB_BASE_NAME}"
+    }
+    options {
+        ansiColor('xterm')
+        buildDiscarder(logRotator(numToKeepStr: '20'))
+        disableConcurrentBuilds()
+        timestamps()
+    }
+    stages {
+        stage('Terraform destroy') {
+            when {
+                expression { params.Recreate == true }
+            }
+            steps {
+                build job: "Terraform destroy", parameters: [[$class: 'StringParameterValue', name: 'Project', value: "${PROJECT_PATH}"]]
+            }
+        }
+
+        stage('Terraform create') {
+            when {
+                expression { params.Recreate == true }
+            }
+            steps {
+                build job: "Terraform create", parameters: [[$class: 'StringParameterValue', name: 'Project', value: "${PROJECT_PATH}"]]
+            }
+        }
+
+        stage('apt update and apt dist-upgrade') {
+            when {
+                expression { params.Recreate == true }
+            }
+            steps {
+                script {
+                    timeout(10) {
+                        build job: "Salt apt update and apt dist-upgrade", parameters: [
+                            [$class: "StringParameterValue", name: "targetname", value: "${NODE}"]
+                        ]
+                    }
+                }
+            }
+        }
+
+        stage('Install package') {
+            steps {
+                script {
+                    build job: "Salt Install Package", parameters: [
+                        [$class: 'StringParameterValue', name: 'targetname', value: "${NODE}"],
+                        [$class: 'StringParameterValue', name: 'pkgname', value: "confd"]
+                    ]
+                }
+            }
+        }
+
+        stage("Install Kibana") {
+            when {
+                expression { params.Recreate == true }
+            }
+            steps {
+                script {
+                    command = """
+                           cd /tmp
+                           echo '"'"'deb [arch=amd64] http://apt.tt.kit/ubuntu-elk-7 stable main  '"'"' | sudo tee -a /etc/apt/sources.list.d/elastic-7.x.list
+                           sudo wget -O - http://apt.tt.kit/ubuntu/apt-gpg-keys/ELK-GPG-KEY.pub | sudo apt-key add -
+                           sudo apt-get update
+                           sudo apt-get install nginx -y
+                           sudo apt-get install kibana=${params.KibanaVersion} -y
+                         """
+                    sh "$SSH riajenk@$NODE '$command'"
+                }
+            }
+        }
+
+        stage('Update conf and start Kibana') {
+            steps {
+                script {
+                    ssh_build_cmd = '''
+                            set -x
+                            set -e
+
+                            cd /tmp/
+
+                            sudo mkdir -p /etc/ssl/nginx/sites-available
+                            sudo bash ./postinst.sh
+                        '''
+                    ELK_HOST = common.host2ip("${NODE}")
+                    sh "$RSYNC -av kibana/confd riajenk@$ELK_HOST:/tmp/"
+                    sh "$RSYNC -av kibana/postinst.sh riajenk@$ELK_HOST:/tmp/"
+                    sh "$SSH riajenk@$ELK_HOST '$ssh_build_cmd'"
+                }
+            }
+        }
+
+        stage('Start Kibana') {
+            steps {
+                script {
+                    ssh_start_kibana = '''
+                        sudo systemctl start kibana
+                    '''
+                    ELK_HOST = common.host2ip("${NODE}")
+                    sh "$SSH riajenk@$ELK_HOST '$ssh_start_kibana'"
+                }
+            }
+        }
+
+        stage('Check Kibana health') {
+            steps {
+                script {
+                    ssh_build_cmd = '''
+                        kibanaHost="http://localhost:5601"
+                        echo "Kibana health check for ${HOSTNAME}:"
+                        n=0
+                        until [ "$n" -ge 60 ]
+                        do
+                           $(curl --output /dev/null --insecure --silent --head --fail "$kibanaHost") && echo "Kibana is up and running" && exit 0 && break
+                           n=$((n+1))
+                           echo "Kibana is unavailable - sleeping"
+                           sleep 10
+                        done
+                        echo "Kibana did not start on time. Exiting ..."
+                        exit 1
+                    '''
+                    ELK_HOST = common.host2ip("${NODE}")
+                    sh "$SSH riajenk@$ELK_HOST '$ssh_build_cmd'"
+                }
+            }
+        }
+    }
+    post {
+        always {
+            cleanWs()
+        }
+        fixed {
+            echo "Build back to normal"
+        }
+    }
+}

kibana/deb/confd/conf.d/default.new.toml

+[template]
+src = "default.new.tmpl"
+dest = "/etc/nginx/sites-available/default.new"
+mode = "0640"
+uid = 0
+gid = 114
+keys = [
+    "jut"
+]

kibana/deb/confd/conf.d/elk-kibana.toml

+[template]
+src = "elk-kibana.tmpl"
+dest = "/etc/logrotate.d/elk-kibana"
+mode = "0644"
+uid = 0
+gid = 0
+keys = [
+    "jut"
+]

kibana/deb/confd/conf.d/kibana.yml.toml

+[template]
+src = "kibana.yml.tmpl"
+dest = "/etc/kibana/kibana.yml"
+mode = "0640"
+uid = 33
+gid = 33
+keys = [
+    "jut"
+]

kibana/deb/confd/templates/default.new.tmpl

+server {
+    listen 80;
+    root /var/www/html;
+
+    location / {
+      proxy_set_header Host $http_host;
+      proxy_set_header X-FRONT-END-HTTPS on;
+      proxy_set_header HTTPS on;
+      proxy_set_header X-Forwarded-For $remote_addr;
+      proxy_set_header X-Forwarded-Proto https;
+      proxy_pass http://127.0.0.1:5601/;
+      proxy_redirect          http:// $scheme://;
+    }
+}

kibana/deb/confd/templates/elk-kibana.tmpl

+/var/log/kibana/*.log {
+  missingok
+  daily
+  size 10M
+  create 0644 kibana kibana
+  rotate 7
+  notifempty
+  sharedscripts
+  notifempty
+  compress
+  postrotate
+    /bin/kill -HUP $(cat /var/run/kibana/kibana.pid 2>/dev/null) 2>/dev/null
+  endscript
+}

kibana/deb/confd/templates/kibana.yml.tmpl

+# Kibana settings
+server.host: "0.0.0.0"
+elasticsearch.hosts: "http://{{getv "/jut/data/elasticHost"}}:9200"
+pid.file: /var/run/kibana/kibana.pid
+logging.dest: /var/log/kibana/kibana.log
+monitoring.enabled: false

kibana/postinst.sh

+#!/bin/bash
+sleep 30
+
+mkdir -p /var/log/kibana
+chown kibana:kibana /var/log/kibana
+
+mkdir -p /var/run/kibana
+chown kibana:kibana /var/run/kibana
+
+mkdir -p /opt/rig
+
+mkdir -p /etc/confd
+mkdir -p /etc/confd/conf.d
+mkdir -p /etc/confd/templates
+mv /tmp/confd/conf.d/* /etc/confd/conf.d/
+mv /tmp/confd/templates/* /etc/confd/templates/
+
+source /etc/vaultcred && confd \
+  -onetime \
+  -prefix $workspace \
+  -backend vault \
+  -node $vaulthost \
+  -auth-type token \
+  -auth-token $vaulttoken
+
+if [ $? -ne 0 ]
+then
+  echo "Failed to apply configuration, use vault_test.sh to verify connection
+   to vault is working. If that is working, check confd templates and
+   configuration. To run it manually use
+  source /etc/vaultcred && confd -onetime -prefix $workspace
+   -backend vault -node $vaulthost -auth-type token
+    -auth-token $vaulttoken
+  to finish installation properly use apt-get -f install"
+  exit 1
+fi
+
+chmod -v o+r /etc/kibana/*.yml
+
+rm -rf /etc/nginx/sites-available/default
+mv /etc/nginx/sites-available/default.new /etc/nginx/sites-available/default
+
+systemctl enable nginx
+systemctl restart nginx
+
+systemctl stop kibana