Elasticsearch indices can be managed using custom scripts when tools like Elastic Curator are unavialable, especially in isolated environments. This approach helps control shard counts, as closed indices do not contribute to the cluste'rs shard limit.

To monitor shard distribution per node, excluding closed shards, use:

curl -s -u 'elastic:password' 'http://node-ip:9200/_cat/shards?v&h=i,s,n' | awk 'NR !=1 { arr[$3]+=$2} END { for (i in arr) print arr[i], i }'

For total shards including closed ones:

curl -s -u 'elastic:password' 'http://node-ip:9200/_cat/allocation?v&h=s,n'

A Bash script automates index closing and deletino based on age. Configure it with cron, e.g., 0 20 * * * /path/to/script.sh &>> /tmp/script.log.

Script Configuration

#!/bin/bash
ES_HOST="node-ip:9200"
RETENTION_MONTHS=3
CLOSE_MONTHS=1
CREDENTIALS="elastic:password"
PATTERN="_project_"

INDEX_LIST=$(curl -s -u "$CREDENTIALS" -XGET "${ES_HOST}/_cat/indices" | grep "$PATTERN" | egrep '(19|20)([0-9]{2})+')

calc_threshold() {
    local months=$1
    date -d "${months} months ago" +%Y-%m-%d
}
CLOSE_CUTOFF=$(calc_threshold $CLOSE_MONTHS)
DELETE_CUTOFF=$(calc_threshold $RETENTION_MONTHS)

parse_index_date() {
    local idx_name=$1
    local granularity="daily"
    local extracted_date=$(echo $idx_name | egrep -o '[0-9]{4}([_.-][0-9]+([_.-][0-9]+)?)?' | tr '[:punct:]' '-')
    [[ $idx_name =~ 'critical_index' ]] && return
    local date_len=${#extracted_date}

    if [ $date_len -lt 5 ]; then
        extracted_date="${extracted_date}-12-31"
        granularity="yearly"
    elif [ $date_len -lt 10 ]; then
        extracted_date="${extracted_date}-28"
        granularity="monthly"
    fi
    local idx_timestamp=$(date -d "${extracted_date}" +%s)
    local current_timestamp=$(date +%s)
    local diff_seconds=$((current_timestamp - idx_timestamp))
    local diff_months=$((diff_seconds / 3600 / 24 / 31))
    local diff_days=$((diff_seconds / 3600 / 24))
    echo "$granularity $idx_timestamp $diff_months $diff_days"
}

close_indices() {
    local open_indices=$(echo "$INDEX_LIST" | grep open | grep -v close | awk '{print $3}')
    for idx in $open_indices; do
        read granularity idx_ts month_diff day_diff <<< $(parse_index_date $idx)
        local close_limit=$((CLOSE_MONTHS * 31))
        if [ $idx_ts -lt $(date -d "$CLOSE_CUTOFF" +%s) ]; then
            if [[ $granularity != "daily" && $month_diff -ge $CLOSE_MONTHS ]] || [[ $granularity == "daily" && $day_diff -ge $close_limit ]]; then
                echo "$(date +%F_%T) Closing index: $idx"
                curl -s -u "$CREDENTIALS" -XPOST "${ES_HOST}/$idx/_close"
                sleep 5
                curl -s -u "$CREDENTIALS" -XPUT "${ES_HOST}/$idx/_settings" -H 'Content-Type: application/json' -d '{"index":{"number_of_replicas":0}}'
                sleep 5
            fi
        fi
    done
    echo
}

delete_indices() {
    local closed_indices=$(echo "$INDEX_LIST" | grep close | awk '{print $3}' | sort)
    for idx in $closed_indices; do
        [[ $idx =~ 'critical_index' ]] && continue
        read granularity idx_ts month_diff day_diff <<< $(parse_index_date $idx)
        local delete_limit=$((RETENTION_MONTHS * 31))
        if [ $idx_ts -lt $(date -d "$DELETE_CUTOFF" +%s) ]; then
            if [[ $granularity != "daily" && $month_diff -ge $RETENTION_MONTHS ]] || [[ $granularity == "daily" && $day_diff -ge $delete_limit ]]; then
                echo "$(date +%F_%T) Deleting index: $idx"
                curl -s -u "$CREDENTIALS" -XDELETE "${ES_HOST}/$idx"
                sleep 5
            fi
        fi
    done
}

purge_annual_data() {
    local annual_limit=12
    local closed_indices=$(echo "$INDEX_LIST" | grep close | awk '{print $3}' | sort)
    for idx in $closed_indices; do
        read granularity idx_ts month_diff day_diff <<< $(parse_index_date $idx)
        local annual_days=$((annual_limit * 31))
        if [ $idx_ts -lt $(date -d "$DELETE_CUTOFF" +%s) ]; then
            if [[ $granularity != "daily" && $month_diff -ge $annual_limit ]] || [[ $granularity == "daily" && $day_diff -ge $annual_days ]]; then
                echo "$(date +%F_%T) Purging annual data: $idx"
                curl -s -u "$CREDENTIALS" -XDELETE "${ES_HOST}/$idx"
                sleep 5
            fi
        fi
    done
}

close_indices
delete_indices
purge_annual_data

Verification After execution, check logs for acknowledgments:

Closing index: monitor_project_2023_09_07
{"acknowledged":true,"shards_acknowledged":true,"indices":{"monitor_project_2023_09_07":{"closed":true}}}
Deleting index: monitor_project_2023_09_07
{"acknowledged":true}