惯性聚合 高效追踪和阅读你感兴趣的博客、新闻、科技资讯
阅读原文 在惯性聚合中打开

推荐订阅源

Forbes - Security
Forbes - Security
A
Arctic Wolf
M
MIT News - Artificial intelligence
T
Threat Research - Cisco Blogs
T
The Exploit Database - CXSecurity.com
C
CERT Recently Published Vulnerability Notes
NISL@THU
NISL@THU
L
Lohrmann on Cybersecurity
Martin Fowler
Martin Fowler
A
About on SuperTechFans
P
Palo Alto Networks Blog
Project Zero
Project Zero
The GitHub Blog
The GitHub Blog
WordPress大学
WordPress大学
Blog — PlanetScale
Blog — PlanetScale
博
博客园_首页
大猫的无限游戏
大猫的无限游戏
Cisco Talos Blog
Cisco Talos Blog
P
Proofpoint News Feed
D
DataBreaches.Net
Cyberwarzone
Cyberwarzone
T
Tor Project blog
IT之家
IT之家
P
Proofpoint News Feed
Help Net Security
Help Net Security
S
Securelist
Threat Intelligence Blog | Flashpoint
Threat Intelligence Blog | Flashpoint
C
CXSECURITY Database RSS Feed - CXSecurity.com
Microsoft Azure Blog
Microsoft Azure Blog
V2EX - 技术
V2EX - 技术
K
Kaspersky official blog
Hugging Face - Blog
Hugging Face - Blog
MongoDB | Blog
MongoDB | Blog
B
Blog
N
News and Events Feed by Topic
The Cloudflare Blog
S
Schneier on Security
P
Privacy & Cybersecurity Law Blog
T
The Blog of Author Tim Ferriss
Recorded Future
Recorded Future
Last Week in AI
Last Week in AI
The Last Watchdog
The Last Watchdog
Hacker News - Newest:
Hacker News - Newest: "LLM"
L
LangChain Blog
I
InfoQ
F
Full Disclosure
The Register - Security
The Register - Security
阮一峰的网络日志
阮一峰的网络日志
H
Hacker News: Front Page
V
V2EX

侯锐的思考与分享

大数据技术的演进：从 Hive、GFS 到 Raft 如何进行站点网络优化 XXL-JOB的部署、搭建与使用 使用NGINX的auth_request进行统一jwt鉴权 使用APISIX解析jwt并获取payload信息 《推荐系统实践》 《上瘾：让用户养成使用习惯的四大产品逻辑》 Kotlin与Java对照手册 ComfyUI的操作与使用 ComfyUI简介 利用whisper为视频自动生成字幕 ffmpeg笔记 如何参与Apache顶级开源项目 使用frp实现内网ssh穿透 安装并使用zsh APISIX的使用 自己动手实现一个可以运行在JVM上的编程语言 使用ANTLR4格式化JSON ANTLR4从入门到实践 ShardingSphere-JDBC介绍 Maven详细介绍
使用Elasticsearch分析腾讯云EO日志
Raymond · 2025-12-27 · via 侯锐的思考与分享

我们参考官方文档使用docker来启动集群，首先下载.env和docker-compose.yml，之后在.env文件中设置es和kibana的密码都是123456，然后设置STACK_VERSION=9.2.3。考虑到数据量比较大，可以提高容器的内存大小，我这里设置了一台8G。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
# Password for the 'elastic' user (at least 6 characters)
ELASTIC_PASSWORD=123456

# Password for the 'kibana_system' user (at least 6 characters)
KIBANA_PASSWORD=123456

# Version of Elastic products
STACK_VERSION=9.2.3

# Set the cluster name
CLUSTER_NAME=elasticsearch-cluster

# Set to 'basic' or 'trial' to automatically start the 30-day trial
LICENSE=basic

# Port to expose Elasticsearch HTTP API to the host
ES_PORT=9200

# Port to expose Kibana to the host
KIBANA_PORT=5601

# Increase or decrease based on the available host memory (in bytes)
MEM_LIMIT=8589934592

# Project namespace (defaults to the current folder name if not set)
COMPOSE_PROJECT_NAME=elasticsearch-project
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
{
  "eo_logs": {
    "aliases": {},
    "mappings": {
      "properties": {
        "ClientIP": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "ClientISP": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "ClientRegion": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "ClientState": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "ContentID": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "EdgeCacheStatus": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "EdgeFunctionSubrequest": {
          "type": "long"
        },
        "EdgeInternalTime": {
          "type": "long"
        },
        "EdgeResponseBodyBytes": {
          "type": "long"
        },
        "EdgeResponseBytes": {
          "type": "long"
        },
        "EdgeResponseStatusCode": {
          "type": "long"
        },
        "EdgeResponseTime": {
          "type": "long"
        },
        "EdgeServerID": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "EdgeServerIP": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "ParentRequestID": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "RemotePort": {
          "type": "long"
        },
        "RequestBytes": {
          "type": "long"
        },
        "RequestHost": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "RequestID": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "RequestMethod": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "RequestProtocol": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "RequestRange": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "RequestReferer": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "RequestStatus": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "RequestTime": {
          "type": "date"
        },
        "RequestUA": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "RequestUrl": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "RequestUrlQueryString": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "_import_time": {
          "type": "date"
        },
        "_source_file": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        }
      }
    },
    "settings": {
      "index": {
        "routing": {
          "allocation": {
            "include": {
              "_tier_preference": "data_content"
            }
          }
        },
        "number_of_shards": "1",
        "provided_name": "eo_logs",
        "creation_date": "1766816305347",
        "number_of_replicas": "1",
        "uuid": "wi9l88cjRh-Kq7lgl4NReg",
        "version": {
          "created": "9039003"
        }
      }
    }
  }
}

然后我们想看指定域名的请求耗时情况（从EdgeOne接收到客户端发起的请求开始，到响应给客户端最后一个字节，整个过程的耗时，对应字段EdgeResponseTime），可以使用如下DSL

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
POST /eo_logs/_search
{
    "size": 0,
    "query": {
        "bool": {
            "filter": [
                {
                    "term": {
                        "RequestHost.keyword": "static.example.com"
                    }
                }
            ]
        }
    },
    "aggs": {
        "edge_response_stats": {
            "stats": {
                "field": "EdgeResponseTime"
            }
        },
        "edge_response_percentiles": {
            "percentiles": {
                "field": "EdgeResponseTime",
                "percents": [
                    50,
                    90,
                    95,
                    99
                ]
            }
        },
        "edge_response_hist": {
            "histogram": {
                "field": "EdgeResponseTime",
                "interval": 50,
                "min_doc_count": 1
            }
        }
    }
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
{
  "took": 3128,
  "timed_out": false,
  "_shards": {
    "total": 1,
    "successful": 1,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": {
      "value": 10000,
      "relation": "gte"
    },
    "max_score": null,
    "hits": []
  },
  "aggregations": {
    "edge_response_percentiles": {
      "values": {
        "50.0": 5.014287434842656,
        "90.0": 25.778307762642324,
        "95.0": 73.78316545752277,
        "99.0": 593.9728031414846
      }
    },
    "edge_response_hist": {
      "buckets": [
        {
          "key": 0.0,
          "doc_count": 25997272
        },
        {
          "key": 50.0,
          "doc_count": 841843
        },
        {
          "key": 100.0,
          "doc_count": 377168
        },
        {
          "key": 150.0,
          "doc_count": 109181
        },
        {
          "key": 200.0,
          "doc_count": 53672
        },
        {
          "key": 250.0,
          "doc_count": 37425
        },
        {
          "key": 300.0,
          "doc_count": 32744
        },
        {
          "key": 350.0,
          "doc_count": 36445
        },
        {
          "key": 400.0,
          "doc_count": 26137
        },
        {
          "key": 450.0,
          "doc_count": 22807
        },
        {
          "key": 500.0,
          "doc_count": 21111
        },
        {
          "key": 550.0,
          "doc_count": 16784
        },
        {
          "key": 600.0,
          "doc_count": 13214
        },
        {
          "key": 650.0,
          "doc_count": 11211
        },
        {
          "key": 700.0,
          "doc_count": 11760
        },
        {
          "key": 750.0,
          "doc_count": 11911
        },
        {
          "key": 800.0,
          "doc_count": 10381
        },
        {
          "key": 850.0,
          "doc_count": 9158
        },
        {
          "key": 900.0,
          "doc_count": 6851
        },
        {
          "key": 950.0,
          "doc_count": 5822
        },
        {
          "key": 1000.0,
          "doc_count": 5195
        },
        ...
      ]
    },
    "edge_response_stats": {
      "count": 27840645,
      "min": 1.0,
      "max": 707706.0,
      "avg": 46.91420216737076,
      "sum": 1.306121648E9
    }
  }
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
POST /eo_logs/_search
{
    "size": 0,
    "query": {
        "bool": {
            "filter": [
                {
                    "term": {
                        "RequestHost.keyword": "static.example.com"
                    }
                },
                {
                    "exists": {
                        "field": "RequestUrl.keyword"
                    }
                },
                {
                    "exists": {
                        "field": "EdgeResponseTime"
                    }
                }
            ]
        }
    },
    "aggs": {
        "by_url": {
            "terms": {
                "field": "RequestUrl.keyword",
                "size": 200,
                "order": {
                    "p95_edge_response[95.0]": "desc"
                }
            },
            "aggs": {
                "p95_edge_response": {
                    "percentiles": {
                        "field": "EdgeResponseTime",
                        "percents": [
                            95
                        ]
                    }
                },
                "avg_edge_response": {
                    "avg": {
                        "field": "EdgeResponseTime"
                    }
                },
                "count_requests": {
                    "value_count": {
                        "field": "EdgeResponseTime"
                    }
                }
            }
        }
    }
}

我们可以针对上面查询到的慢速URL去做特定的优化和缓存预热。只是，上面的这个DSL不够严谨，因为单纯使用请求时间来判断速度快慢是不足够的，请求时间也会受到资源大小的影响。因此，我们使用资源的大小比上请求耗时，这个就代表这个资源的下载速度，之后我们从小到大排序，就可以知道哪些资源可能会下载比较慢了。具体DSL如下

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
POST /eo_logs/_search
{
    "size": 0,
    "query": {
        "bool": {
            "filter": [
                {
                    "term": {
                        "RequestHost.keyword": "static.example.com"
                    }
                },
                {
                    "exists": {
                        "field": "RequestUrl.keyword"
                    }
                },
                {
                    "exists": {
                        "field": "EdgeResponseTime"
                    }
                },
                {
                    "exists": {
                        "field": "EdgeResponseBodyBytes"
                    }
                },
                {
                    "range": {
                        "EdgeResponseBodyBytes": {
                            "gt": 0
                        }
                    }
                },
                {
                    "range": {
                        "EdgeResponseTime": {
                            "gt": 0
                        }
                    }
                }
            ]
        }
    },
    "aggs": {
        "by_url": {
            "terms": {
                "field": "RequestUrl.keyword",
                "size": 2000,
                "order": {
                    "avg_kbps": "asc"
                }
            },
            "aggs": {
                "avg_kbps": {
                    "avg": {
                        "script": {
                            "lang": "painless",
                            "source": "double b = doc['EdgeResponseBodyBytes'].value; double t = doc['EdgeResponseTime'].value; return (b / t) * (1000.0 / 1024.0);"
                        }
                    }
                },
                "p95_kbps": {
                    "percentiles": {
                        "script": {
                            "lang": "painless",
                            "source": "double b = doc['EdgeResponseBodyBytes'].value; double t = doc['EdgeResponseTime'].value; return (b / t) * (1000.0 / 1024.0);"
                        },
                        "percents": [
                            95
                        ]
                    }
                },
                "avg_time_ms": {
                    "avg": {
                        "field": "EdgeResponseTime"
                    }
                },
                "avg_body_bytes": {
                    "avg": {
                        "field": "EdgeResponseBodyBytes"
                    }
                },
                "req_count": {
                    "value_count": {
                        "field": "EdgeResponseTime"
                    }
                }
            }
        }
    }
}

此内容由惯性聚合(RSS阅读器)自动聚合整理，仅供阅读参考。 原文来自 — 版权归原作者所有。