ElasticSearch Query Programatically

1#!/usr/bin/env node
2'use strict';
3const axios = require('axios');
4const { format, subMinutes } = require('date-fns');
5
6const processResult = (rawResult) => {
7  const { data } = rawResult;
8  const { hits } = data;
9  const iterableHits = hits.hits;
10  const errorsList = iterableHits.map((iterableHit) => {
11    const { _source } = iterableHit;
12    const { log, stream, kubernetes } = _source;
13    return {
14      log,
15      stream,
16      service: `${kubernetes.container_name} : ${kubernetes.pod_name}`
17    };
18  });
19  return errorsList;
20};
21
22const sendToSlack = (errorsList) => {
23  console.log('errorsList', errorsList);
24  const allSlackMessages = errorsList.map((errorsListItem) => {
25    const stringifiedError = JSON.stringify(errorsListItem);
26    const postBodyPayload = { text: stringifiedError };
27    const uri = `${process.env.slackHook}`;
28    return axios.post(uri, postBodyPayload);
29  });
30  return Promise.all(allSlackMessages);
31};
32
33module.exports.run = async (event) => {
34  const day = format(new Date(), 'MM.dd');
35  const year = format(new Date(), 'yyyy');
36  const lte = new Date().toISOString();
37  const gte = subMinutes(new Date(), 6).toISOString();
38  const postBodyQuery = {
39    size: 500,
40    query: {
41      bool: {
42        must: [
43          {
44            range: {
45              '@timestamp': {
46                format: 'strict_date_optional_time',
47                gte,
48                lte
49              }
50            }
51          }
52        ],
53        filter: [
54          {
55            bool: {
56              filter: [
57                {
58                  bool: {
59                    should: [
60                      {
61                        match_phrase: {
62                          'kubernetes.container_name': 'migration*'
63                        }
64                      }
65                    ],
66                    minimum_should_match: 1
67                  }
68                },
69                {
70                  bool: {
71                    filter: [
72                      {
73                        bool: {
74                          must_not: {
75                            bool: {
76                              should: [
77                                {
78                                  match_phrase: {
79                                    log: 'kube-probe*'
80                                  }
81                                }
82                              ],
83                              minimum_should_match: 1
84                            }
85                          }
86                        }
87                      },
88                      {
89                        bool: {
90                          filter: [
91                            {
92                              bool: {
93                                must_not: {
94                                  bool: {
95                                    should: [
96                                      {
97                                        match_phrase: {
98                                          log: 'health*'
99                                        }
100                                      }
101                                    ],
102                                    minimum_should_match: 1
103                                  }
104                                }
105                              }
106                            },
107                            {
108                              bool: {
109                                filter: [
110                                  {
111                                    bool: {
112                                      must_not: {
113                                        bool: {
114                                          should: [
115                                            {
116                                              match_phrase: {
117                                                log: 'broacasting*'
118                                              }
119                                            }
120                                          ],
121                                          minimum_should_match: 1
122                                        }
123                                      }
124                                    }
125                                  },
126                                  {
127                                    bool: {
128                                      filter: [
129                                        {
130                                          bool: {
131                                            must_not: {
132                                              bool: {
133                                                should: [
134                                                  {
135                                                    match_phrase: {
136                                                      log: 'Origin check*'
137                                                    }
138                                                  }
139                                                ],
140                                                minimum_should_match: 1
141                                              }
142                                            }
143                                          }
144                                        },
145                                        {
146                                          bool: {
147                                            filter: [
148                                              {
149                                                bool: {
150                                                  should: [
151                                                    {
152                                                      match_phrase: {
153                                                        log: '[31merror'
154                                                      }
155                                                    }
156                                                  ],
157                                                  minimum_should_match: 1
158                                                }
159                                              },
160                                              {
161                                                bool: {
162                                                  should: [
163                                                    {
164                                                      match_phrase: {
165                                                        _index: `logstash-${year}.${day}`
166                                                      }
167                                                    }
168                                                  ],
169                                                  minimum_should_match: 1
170                                                }
171                                              }
172                                            ]
173                                          }
174                                        }
175                                      ]
176                                    }
177                                  }
178                                ]
179                              }
180                            }
181                          ]
182                        }
183                      }
184                    ]
185                  }
186                }
187              ]
188            }
189          }
190        ],
191        should: [],
192        must_not: []
193      }
194    }
195  };
196  const uri = `${process.env.elasticEndpoint}/_search`;
197
198  try {
199    const axiosProps = { headers: { Authorization: `Basic ${process.env.elasticAuthentication}` } };
200    // Errors
201    const logs = await axios.post(uri, postBodyQuery, axiosProps);
202    const allErrors = processResult(logs);
203    await sendToSlack(allErrors);
204    // Terminate successfully
205    return {
206      statusCode: 200,
207      body: JSON.stringify(
208        {
209          message: 'Logs Processed!',
210          input: event
211        },
212        null,
213        2
214      )
215    };
216  } catch (err) {
217    console.log('err', err);
218    return {
219      statusCode: 500,
220      body: JSON.stringify(
221        {
222          message: 'Failed to read logs!',
223          input: event
224        },
225        null,
226        2
227      )
228    };
229  }
230};