Elasticsearch Cluster restart and upgrade

Introduction

As part of operational maintenance your elastic cluster needs updating and the running OS may need kernel/security updates as well

Upgrading your Cluster

Disable shard allocation

PUT _cluster/settings
{
  "persistent": {
    "cluster.routing.allocation.enable": "primaries"
  }
}

#OUTPUT

{
  "acknowledged" : true,
  "persistent" : {
    "cluster" : {
      "routing" : {
        "allocation" : {
          "enable" : "primaries"
        }
      }
    }
  },
  "transient" : { }
}

Disable shard allocation

POST _flush/synced

#OUTPUT (excerpt)

{
  "_shards" : {
    "total" : 199,
    "successful" : 117,
    "failed" : 2
  },
  ".monitoring-logstash-7-2021.06.27" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0
  },
  ".monitoring-logstash-7-2021.06.26" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0
  },
  ".triggered_watches" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0
  },
  ".monitoring-logstash-7-2021.06.25" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0
  },
  ".monitoring-logstash-7-2021.06.24" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0
  },
  ".monitoring-logstash-7-2021.06.23" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0
  },
  ".monitoring-logstash-7-2021.06.22" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0

Shutdown All nodes

# shutdown all nodes with the following sequence
#
# 1) Logstash
sudo systemctl stop logstash.service

# 2) Kibana
sudo systemctl stop kibana.service

# 3) Elasticsearch
sudo systemctl stop elasticsearch.service

POST _flush/synced

OS update (if needed)

# Cluster is running on Centos
#
sudo yum update

# Reboot if required
sudo reboot

Upgrade Elastic

# Upgrade elastic in the following sequence
# local installation (i.e. rpm packages download)
#
# https://www.elastic.co/guide/en/elasticsearch/reference/7.13/rpm.html#install-rpm
# https://www.elastic.co/guide/en/kibana/7.13/install.html
# https://www.elastic.co/guide/en/logstash/7.13/installing-logstash.html

# 1) Elasticsearch
     sudo rpm --upgrade /home/rmacatba/elasticsearch-xx.x.x.rpm
     sudo systemctl daemon-reload
     sudo systemctl start elasticsearch.service

# 2) Kibana
     sudo rpm --upgrade /home/rmacatba/kibana-xx.x.x.rpm
     sudo systemctl start kibana.service
     sudo systemctl daemon-reload
     
# 3) logstash
     sudo rpm --upgrade /home/rmacatba/logstash-xx.x.x.rpm
     sudo rpm --upgrade /home/rmacatba/logstash-7.13.2-x86_64.rpm
     systemctl start logstash.service
     sudo systemctl daemon-reload