Go Elasticsearch 快速入门
相关文档
聚合
例一
例二
Index Mapping:
{
"mappings": {
"properties": {
"userid": {
"type": "keyword"
}
}
}
}Search Query:
{
"size": 0,
"aggs": {
"userId": {
"terms": {
"field": "userid"
},
"aggs": {
"the_filter": {
"bucket_selector": {
"buckets_path": {
"the_doc_count": "_count"
},
"script": "params.the_doc_count == 3" //取桶聚合值等于3的
}
}
}
}
}
}Search Result:
"aggregations": {
"userId": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": "1", // this denotes the userid having count==3
"doc_count": 3
}
]
}例三
mapping
{
"routing_num_shards":768,
"settings":{
"index":{
"mapping":{
"total_fields":{
"limit":"10000"
}
},
"number_of_shards":"6",
"max_result_window":"2000000",
"number_of_replicas":"0"
}
},
"mappings":{
"_doc":{
"properties":{
"country":{
"type":"keyword"
},
"cloudPlatform":{
"type":"keyword"
},
"beginAt":{
"type":"text",
"fields":{
"keyword":{
"ignore_above":256,
"type":"keyword"
}
}
},
"os":{
"type":"keyword"
},
"city":{
"type":"keyword"
},
"hostIP":{
"type":"ip"
},
"nodeType":{
"type":"keyword"
},
"type":{
"type":"short"
},
"version":{
"type":"keyword"
},
"isFullNode":{
"type":"boolean"
},
"netProtocol":{
"type":"short"
},
"createdAt":{
"format":"yyyy-MM-dd HH:mm:ss",
"type":"date"
},
"srcAddr":{
"type":"keyword"
},
"province":{
"type":"keyword"
},
"hostPort":{
"type":"integer"
},
"addr":{
"type":"keyword"
},
"nodeID":{
"type":"keyword"
},
"user":{
"type":"keyword"
},
"height":{
"type":"integer"
},
"status":{
"type":"short"
},
"updatedAt":{
"format":"yyyy-MM-dd HH:mm:ss",
"type":"date"
}
}
}
}
}data：
POST certdata-blockchainnodedetect/_doc
{
"updatedAt": "2023-02-17 09:53:23",
"createdAt": "2023-02-17 09:53:23",
"nodeID": "blockchainDetection_btcd_1001",
"addr": "64.44.118.194:48442",
"srcAddr": "176.126.167.10:8334",
"status": 1,
"type": 2,
"version": "0",
"user": "",
"os": "",
"cloudPlatform": "",
"nodeType": "",
"isFullNode": false,
"height": 0,
"hostIP": "64.44.118.194",
"hostPort": 48442,
"netProtocol": 1,
"country": "未知地区",
"province": "未知地区",
"city": "未知地区"
}Search Query:
GET certdata-blockchainnodedetect/_search
{
"size": 0,
"aggs": {
"aggIp": {
"terms": {
"field": "hostIP",
"size": 2,
"min_doc_count": 2 // 取doc_count>2的桶
},
"aggs": {
"aggType": {
"terms": {
"field": "type",
"size": 2,
"min_doc_count": 1 // 取doc_count>1的桶
}
}
}
}
},
"track_total_hits":true
}Search Result:
"aggregations" : {
"aggIp" : {
"doc_count_error_upper_bound" : 29,
"sum_other_doc_count" : 145467,
"buckets" : [
{
"key" : "64.44.118.194",
"doc_count" : 31,
"aggType" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 0,
"buckets" : [
{
"key" : 1,
"doc_count" : 30
},
{
"key" : 2,
"doc_count" : 1
}
]
}
},
{
"key" : "107.175.102.74",
"doc_count" : 29,
"aggType" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 0,
"buckets" : [
{
"key" : 1,
"doc_count" : 29
}
]
}
}
]
}
}例四
ES操作之总和桶聚合(Sum Bucket Aggregation)
使用场景: 获取某分组条件下所有桶的指定度量的和
比如: 根据某个条件分组,获取前1000条数据出现的数量和.
https://www.cnblogs.com/nysd/p/12858355.html
例五
elasticsearch 聚合分桶之后，获取桶的总数及统计
https://www.codenong.com/c3130039/
例六
查时间5年之内 && 国家为GB && 类型为软件
聚合一级分类
子聚合 二级分类、产品、公司
mapping
{
"test" : {
"mappings" : {
"properties" : {
"asn" : {
"dynamic" : "true",
"properties" : {
"number" : {
"type" : "long"
},
"organization" : {
"type" : "keyword"
}
}
},
"banner" : {
"type" : "text"
},
"base_protocol" : {
"type" : "keyword"
},
"body" : {
"type" : "text"
},
"cert" : {
"type" : "text"
},
"certs" : {
"dynamic" : "true",
"properties" : {
"cert_date" : {
"type" : "date",
"format" : "yyyy-MM-dd HH:mm:ss"
},
"cert_num" : {
"type" : "integer"
},
"domain" : {
"type" : "keyword"
},
"is_valid" : {
"type" : "boolean"
},
"issuer_cn" : {
"type" : "keyword"
},
"issuer_cns" : {
"type" : "keyword"
},
"issuer_org" : {
"type" : "keyword"
},
"not_after" : {
"type" : "date",
"format" : "yyyy-MM-dd HH:mm:ss"
},
"not_before" : {
"type" : "date",
"format" : "yyyy-MM-dd HH:mm:ss"
},
"sig_alth" : {
"type" : "keyword"
},
"sn" : {
"type" : "keyword"
},
"subject_cn" : {
"type" : "keyword"
},
"subject_cns" : {
"type" : "keyword"
},
"subject_org" : {
"type" : "keyword"
},
"valid_type" : {
"type" : "keyword"
}
}
},
"create_at" : {
"type" : "date",
"format" : "yyyy-MM-dd HH:mm:ss"
},
"critical" : {
"dynamic" : "true",
"properties" : {
"category" : {
"type" : "keyword"
},
"parent_category" : {
"type" : "keyword"
}
}
},
"dbs" : {
"dynamic" : "true",
"properties" : {
"count" : {
"type" : "long"
},
"db_size" : {
"type" : "long"
},
"name" : {
"type" : "keyword"
},
"records" : {
"type" : "long"
},
"version" : {
"type" : "keyword"
}
}
},
"domain" : {
"type" : "keyword"
},
"favicon" : {
"dynamic" : "true",
"properties" : {
"base64" : {
"type" : "text"
},
"hash" : {
"type" : "text"
},
"url" : {
"type" : "keyword"
}
}
},
"geoip" : {
"dynamic" : "true",
"properties" : {
"city" : {
"type" : "keyword"
},
"continent" : {
"type" : "keyword"
},
"coordinate" : {
"type" : "geo_point"
},
"country" : {
"type" : "keyword"
},
"country_name" : {
"type" : "keyword"
},
"isp" : {
"type" : "keyword"
},
"latitude" : {
"type" : "double"
},
"longitude" : {
"type" : "double"
},
"region" : {
"type" : "keyword"
},
"timezone" : {
"type" : "keyword"
}
}
},
"header" : {
"type" : "text"
},
"host" : {
"type" : "keyword"
},
"hostnames" : {
"type" : "keyword"
},
"id" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"ip" : {
"type" : "ip"
},
"os" : {
"type" : "keyword"
},
"port" : {
"type" : "integer"
},
"products" : {
"type" : "nested",
"dynamic" : "true",
"properties" : {
"category" : {
"type" : "keyword"
},
"company" : {
"type" : "keyword"
},
"level" : {
"type" : "integer"
},
"parent_category" : {
"type" : "keyword"
},
"product" : {
"type" : "keyword"
},
"softhard" : {
"type" : "keyword"
},
"version" : {
"type" : "keyword"
}
}
},
"protocol" : {
"type" : "keyword"
},
"server" : {
"type" : "text"
},
"subdomain" : {
"type" : "keyword"
},
"type" : {
"type" : "keyword"
},
"update_at" : {
"type" : "date",
"format" : "yyyy-MM-dd HH:mm:ss"
},
"vulnerabilities" : {
"type" : "nested",
"dynamic" : "true",
"properties" : {
"cnnvd_id" : {
"type" : "keyword"
},
"cnvd_id" : {
"type" : "keyword"
},
"cve_id" : {
"type" : "keyword"
},
"third_id" : {
"type" : "keyword"
},
"vul_id" : {
"type" : "keyword"
},
"vul_level" : {
"type" : "integer"
},
"vul_name" : {
"type" : "keyword"
},
"vul_scan_at" : {
"type" : "date",
"format" : "yyyy-MM-dd HH:mm:ss"
},
"vul_type" : {
"type" : "keyword"
},
"vul_url" : {
"type" : "keyword"
}
}
}
}
}
}
}数据
{
"_index" : "test",
"_type" : "_doc",
"_id" : "147.20.51.63:47413",
"_score" : 1.0,
"_source" : {
"id" : "147.20.51.63:47413",
"create_at" : "2023-03-08 10:40:15",
"update_at" : "2023-03-08 10:40:15",
"type" : "zoomeye",
"ip" : "147.20.51.63",
"port" : 47413,
"protocol" : "ssh",
"base_protocol" : "",
"geoip" : {
"isp" : "ProPublica",
"city" : "Baton Rouge",
"region" : "Tucson",
"country" : "MU",
"country_name" : "Western Sahara",
"continent" : "Europe",
"timezone" : "E. South America Standard Time",
"longitude" : -160.311844,
"latitude" : 59.053322
},
"host" : "directsolutions.info",
"hostnames" : [
"seniore-markets.name"
],
"domain" : [
"futureout-of-the-box.name"
],
"subdomain" : [
"globalrelationships.io"
],
"os" : [
"Magentakey",
"humanscale.net"
],
"server" : "MediumVioletRedwater 2.12.8",
"header" : "",
"banner" : "",
"body" : "",
"cert" : "",
"certs" : {
"cert_date" : "1908-09-18 14:56:22",
"cert_num" : 62116,
"domain" : "regionalmethodologies.com",
"is_valid" : true,
"issuer_cn" : "Loqate, Inc.",
"issuer_cns" : [
"Abt Associates"
],
"issuer_org" : "PowerAdvocate",
"not_after" : "1987-01-25 17:11:33",
"not_before" : "2023-05-05 22:09:38",
"sig_alth" : "476798901",
"sn" : "792605713",
"subject_cn" : "Deloitte",
"subject_cns" : [
"Aidin"
],
"subject_org" : "FarmLogs",
"valid_type" : "banh mi"
},
"critical" : null,
"asn" : {
"number" : "363979346",
"organization" : "Ez-XBRL"
},
"products" : [
{
"product" : "DarkCyanpod",
"level" : 9,
"category" : "MistyRosehost",
"parent_category" : "Dinosaurhad",
"softhard" : "2",
"company" : "Equilar",
"version" : "4.7.1"
}
],
"dbs" : {
"name" : "Orangelie",
"version" : "4.5.20",
"count" : 3277458818733511498,
"db_size" : -4909779963776056562,
"records" : 6346730448776087845
},
"favicon" : {
"base64" : "If we deconstruct the matrix, we can get to the HDD hard drive through the multi-byte HTTP protocol!",
"hash" : "Overriding the program won't do anything, we need to bypass the primary IB bandwidth!",
"url" : "https://www.forwardenhance.net/strategic"
},
"vulnerabilities" : null
}
}查询
# 查时间5年之内 && 国家为GB && 类型为软件
# 聚合一级分类
# 子聚合 二级分类、产品、公司
GET test/_search
{
"size": 0,
"query": {
"constant_score": {
"boost": 1.2,
"filter": {
"bool": {
"must": [
{
"range": {
"update_at": {
"from": "now-5y",
"include_lower": true,
"include_upper": true,
"to": null
}
}
},
{
"term": {
"geoip.country": "GB"
}
},
{
"nested": {
"path": "products",
"query": {
"bool": {
"must": [
{
"term": {
"products.softhard": {
"value": "2"
}
}
}
]
}
}
}
}
]
}
}
}
},
"aggs": {
"productsGroup": {
"nested": {
"path": "products"
},
"aggs": {
"first_category": {
"terms": {
"field": "products.parent_category",
"order": [
{
"_count": "desc"
}
],
"size": 50
},
"aggs": {
"company": {
"terms": {
"field": "products.company",
"order": [
{
"_count": "desc"
}
],
"size": 50
}
},
"product": {
"terms": {
"field": "products.product",
"order": [
{
"_count": "desc"
}
],
"size": 50
}
},
"second_category": {
"terms": {
"field": "products.category",
"order": [
{
"_count": "desc"
}
],
"size": 50
}
}
}
}
}
}
}
}
go代码片断
func (m *metaDataService) WorldAllDataService(ctx context.Context, in *model.WorldAllDataInput) (resp *elasticCore.SearchResult, err error) {
ctx, span := gtrace.NewSpan(ctx, "MetaData.WorldAllDataService")
defer span.End()
// 开放端口的城市分布
filters := []*Filters{
&Filters{
Qu: "update_at",
Wildcard: ">=",
T: "now-5y",
},
}
if len(in.CateCountryCode) > 0 {
filters = append(filters, &Filters{
Qu: "geoip.country",
Wildcard: "",
T: strings.ToUpper(in.CateCountryCode),
})
}
// 组装查询条件
query := elasticCore.NewBoolQuery()
query = buildFilters(ctx, query, filters)
query = query.Must(
elasticCore.NewNestedQuery(
"products",
elasticCore.NewBoolQuery().
Must(
elasticCore.NewTermQuery("products.softhard", 2),
),
),
)
q := elasticCore.NewConstantScoreQuery(query).Boost(1.2)
sourceQ2, _ := q.Source()
fofacore.PrintQuery(sourceQ2)
// 聚合
firstCategoryAgg := elasticCore.NewTermsAggregation().Field("products.parent_category").Size(50).OrderByCountDesc().
SubAggregation("second_category", elasticCore.NewTermsAggregation().Field("products.category").Size(50).OrderByCountDesc()).
SubAggregation("company", elasticCore.NewTermsAggregation().Field("products.company").Size(50).OrderByCountDesc()).
SubAggregation("product", elasticCore.NewTermsAggregation().Field("products.product").Size(50).OrderByCountDesc())
nestedAgg := elasticCore.NewNestedAggregation().Path("products").
SubAggregation("first_category", firstCategoryAgg)
// print esquery
sourceQ1, _ := nestedAgg.Source()
fofacore.PrintQuery(sourceQ1)
// 查询结果
res, err := EsClient(string(ast.EsMetadata)).
Size(0).
Query(q).
Aggregation("productsGroup", nestedAgg).
Pretty(true).
Do(context.Background())
//glog.Info(ctx, "res:", res)
// 返回结果
return res, err
}
// 组装参数
func buildFilters(ctx context.Context, query *elasticCore.BoolQuery, filters []*Filters) (filersQuery *elasticCore.BoolQuery) {
for i := range filters {
if filters[i].Qu != "" || filters[i].T != "" {
switch filters[i].Wildcard {
case "=":
query = query.Must(elasticCore.NewTermQuery(filters[i].Qu, filters[i].T))
case "in":
values := filters[i].T.([]string)
tmpInter := make([]interface{}, 0)
for v := range values {
tmpInter = append(tmpInter, values[v])
}
query = query.Must(elasticCore.NewTermsQuery(filters[i].Qu, tmpInter...))
case "!=":
query = query.MustNot(elasticCore.NewTermQuery(filters[i].Qu, filters[i].T))
case ">=":
query = query.Must(elasticCore.NewRangeQuery(filters[i].Qu).Gte(filters[i].T))
case "<=":
query = query.Must(elasticCore.NewRangeQuery(filters[i].Qu).Lte(filters[i].T))
case "range":
// filters[i].T = "now-5y,now"
ts := strings.Split(filters[i].T.(string), ",")
query = query.Must(elasticCore.NewRangeQuery(filters[i].Qu).Gte(ts[0]).Lte(ts[1]))
default:
query = query.Must(elasticCore.NewTermQuery(filters[i].Qu, filters[i].T))
}
}
}
return query
}
// PrintQuery 自定义es query打印函数
func PrintQuery(src interface{}) {
log.Println("*****")
data, err := json.MarshalIndent(src, "", " ")
if err != nil {
panic(err)
}
log.Println(string(data))
}
type Filters struct {
Qu string // 字段名 如： is_ipv6
Wildcard string //通配符 如： “=” “!=” ">=" "<=" "in"
T interface{} //搜索值 如： false， 当Wildcard 为 range时 T= "now-5y,now",
}
结果
{
"took" : 3,
"timed_out" : false,
"_shards" : {
"total" : 6,
"successful" : 6,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 1,
"relation" : "eq"
},
"max_score" : null,
"hits" : [ ]
},
"aggregations" : {
"productsGroup" : {
"doc_count" : 1,
"first_category" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 0,
"buckets" : [
{
"key" : "Eleganceare",
"doc_count" : 1,
"product" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 0,
"buckets" : [
{
"key" : "DarkVioletpack",
"doc_count" : 1
}
]
},
"second_category" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 0,
"buckets" : [
{
"key" : "PeachPuffdentist",
"doc_count" : 1
}
]
},
"company" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 0,
"buckets" : [
{
"key" : "AllState Insurance Group",
"doc_count" : 1
}
]
}
}
]
}
}
}
}
例七
数据同例六
要求：查五年以内 domain去重后的总数
esQuery查询
GET metadata/_search
{
"size": 0,
"query": {
"bool": {
"must": {
"range": {
"update_at": {
"from": "now-5y",
"include_lower": true,
"include_upper": true,
"to": null
}
}
}
}
},
"aggs": {
"domain": {
"cardinality": {
"field": "domain"
, "precision_threshold": 4000
}
}
}
}go代码片断
//DomainTotalService domain去重总数
func (m *metaDataService) DomainTotalService(ctx context.Context, in *model.DomainTotalInput) (cardinalityCount int, err error) {
ctx, span := gtrace.NewSpan(ctx, "MetaData.DomainTotalService")
defer span.End()
span.SetAttributes(
attribute.String("MetaData.DomainTotalService.CateCountryCode", in.CateCountryCode),
)
//var key = "index_fofa_world_CS_total"
//if len(in.CateCountryCode) > 0 {
// key += in.CateCountryCode
//}
// 开放端口的城市分布
filters := []*Filters{
&Filters{
Qu: "update_at",
Wildcard: ">=",
T: "now-5y",
},
}
if len(in.CateCountryCode) > 0 {
filters = append(filters, &Filters{
Qu: "geoip.country",
Wildcard: "",
T: strings.ToUpper(in.CateCountryCode),
})
}
cardinalityCount, err = m.GetCardinalityAggs(ctx, "domain", filters)
glog.Info(ctx, "cardinalityCount:", cardinalityCount)
// 返回结果
return cardinalityCount, err
}
// GetCardinalityAggs 聚合数据(去重)总数
func (m *metaDataService) GetCardinalityAggs(ctx context.Context, aggField string, filters []*Filters) (groupByNestedCount int, err error) {
// 获取去重的字段数量（通用）
cardinalityAggCount := elasticCore.NewCardinalityAggregation().
Field(aggField).
PrecisionThreshold(4000)
//Rehash(true)
// print esquery
sourceQ1, _ := cardinalityAggCount.Source()
fofacore.PrintQuery(sourceQ1)
query := elasticCore.NewBoolQuery()
// 组装查询条件
query = buildFilters(ctx, query, filters)
q := elasticCore.NewConstantScoreQuery(query).Boost(1.2)
sourceQ2, _ := q.Source()
fofacore.PrintQuery(sourceQ2)
// 查询结果
res, err := EsClient(string(ast.EsMetadata)).
Size(0).
Query(q).
Aggregation("cardinalityCount", cardinalityAggCount).
Pretty(true).
Do(context.Background())
// 取数据
aggsGroup, _ := res.Aggregations.Terms("cardinalityCount")
for _, bucket := range aggsGroup.Aggregations {
marshalJSON, _ := bucket.MarshalJSON()
groupByNestedCount, err = strconv.Atoi(string(marshalJSON))
}
return
}结果
{
"took" : 1,
"timed_out" : false,
"_shards" : {
"total" : 6,
"successful" : 6,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 53,
"relation" : "eq"
},
"max_score" : null,
"hits" : [ ]
},
"aggregations" : {
"companyGroup" : {
"doc_count" : 53,
"company" : {
"value" : 0
}
}
}
}
例八
数据同例六
company为nested属性
要求：查五年以内company去重后的总数
esQuery查询
GET metadata/_search
{
"size": 0,
"query": {
"bool": {
"must": {
"range": {
"update_at": {
"from": "now-5y",
"include_lower": true,
"include_upper": true,
"to": null
}
}
}
}
},
"aggs": {
"companyGroup": {
"nested": {
"path": "products"
},
"aggs": {
"company": {
"cardinality": {
"field": "products.company",
"precision_threshold": 4000
}
}
}
}
}
}go代码断片
//WorldCsTotalService 当前区域厂商总数 company 去重总数
func (m *metaDataService) WorldCsTotalService(ctx context.Context, in *model.WorldCsTotalResInput) (cardinalityCount int, err error) {
ctx, span := gtrace.NewSpan(ctx, "MetaData.WorldCsTotalService")
defer span.End()
span.SetAttributes(
attribute.String("MetaData.WorldCsTotalService.CateCountryCode", in.CateCountryCode),
)
//var key = "index_fofa_world_CS_total"
//if len(in.CateCountryCode) > 0 {
// key += in.CateCountryCode
//}
// 开放端口的城市分布
filters := []*Filters{
&Filters{
Qu: "update_at",
Wildcard: ">=",
T: "now-5y",
},
}
if len(in.CateCountryCode) > 0 {
filters = append(filters, &Filters{
Qu: "geoip.country",
Wildcard: "",
T: strings.ToUpper(in.CateCountryCode),
})
}
cardinalityCount, err = m.GetCardinalityNestedAggs(ctx, "products.company", filters)
glog.Info(ctx, "cardinalityCount:", cardinalityCount)
// 返回结果
return cardinalityCount, err
}
// GetCardinalityNestedAggs Nested的聚合数据(去重)总数
func (m *metaDataService) GetCardinalityNestedAggs(ctx context.Context, aggField string, filters []*Filters) (groupByNestedCount int, err error) {
subAgg := elasticCore.NewCardinalityAggregation().
Field(aggField).
PrecisionThreshold(4000)
// Nested聚合
sn := elasticCore.NewNestedAggregation()
sn.Path("products")
sn.SubAggregation("group_by_nested", subAgg)
// print esquery
sourceQ1, _ := sn.Source()
fofacore.PrintQuery(sourceQ1)
query := elasticCore.NewBoolQuery()
// 组装查询条件
query = buildFilters(ctx, query, filters)
q := elasticCore.NewConstantScoreQuery(query).Boost(1.2)
sourceQ2, _ := q.Source()
fofacore.PrintQuery(sourceQ2)
// 查询结果
res, err := EsClient(string(ast.EsMetadata)).
Size(0).
Query(q).
Aggregation("aggs", sn).
Pretty(true).
Do(context.Background())
// 取数据
aggsGroup, _ := res.Aggregations.Terms("aggs")
groupByNestedAgg, _ := aggsGroup.Aggregations.Terms("group_by_nested")
for _, bucket := range groupByNestedAgg.Aggregations {
marshalJSON, _ := bucket.MarshalJSON()
groupByNestedCount, err = strconv.Atoi(string(marshalJSON))
}
return
}结果
{
"took" : 1,
"timed_out" : false,
"_shards" : {
"total" : 6,
"successful" : 6,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 53,
"relation" : "eq"
},
"max_score" : null,
"hits" : [ ]
},
"aggregations" : {
"companyGroup" : {
"doc_count" : 53,
"company" : {
"value" : 47
}
}
}
}
例九
sum的value就是分组的os桶的doc_count之和
DSL
GET metadata/_search
{
"size": 0,
"query": {
"constant_score": {
"filter": {
"bool": {
"must": [
{
"term": {
"critical.parent_category": "市政"
}
},
{
"range": {
"update_at": {
"from": "now-5y",
"include_lower": true,
"include_upper": true,
"to": null
}
}
}
]
}
},
"boost": 1.2
}
},
"aggs": {
"port": {
"terms": {
"field": "port",
"size": 5
}
},
"protocol": {
"terms": {
"field": "protocol",
"size": 5
}
},
"os": {
"terms": {
"field": "os",
"size": 10
}
},
"sum":{
"sum_bucket": {
"buckets_path": "os>_count"
}
},
"country": {
"terms": {
"field": "geoip.country_name",
"size": 10
}
},
"city": {
"terms": {
"field": "geoip.city",
"size": 10
}
}
}
}结果：
{
"took" : 4,
"timed_out" : false,
"_shards" : {
"total" : 6,
"successful" : 6,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 2,
"relation" : "eq"
},
"max_score" : null,
"hits" : [ ]
},
"aggregations" : {
"country" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 0,
"buckets" : [
{
"key" : "Angola",
"doc_count" : 2
}
]
},
"protocol" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 0,
"buckets" : [
{
"key" : "ssh",
"doc_count" : 2
}
]
},
"os" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 0,
"buckets" : [
{
"key" : "Pairhad",
"doc_count" : 2
},
{
"key" : "seniormarkets.io",
"doc_count" : 2
}
]
},
"city" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 0,
"buckets" : [
{
"key" : "Boise",
"doc_count" : 2
}
]
},
"port" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 0,
"buckets" : [
{
"key" : 53520,
"doc_count" : 2
}
]
},
"sum" : {
"value" : 4.0
}
}
}
例十
sum的value就是分组的first_category桶的doc_count之和
DSL
GET metadata/_search
{
"size": 0,
"query": {
"constant_score": {
"filter": {
"bool": {
"must": [
{
"range": {
"update_at": {
"from": "now-5y",
"include_lower": true,
"include_upper": true,
"to": null
}
}
}
]
}
},
"boost": 1.2
}
},
"aggs": {
"productsGroup": {
"nested": {
"path": "products"
},
"aggs": {
"parent_category": {
"terms": {
"field": "products.parent_category",
"size": 50
}
},
"sum": {
"sum_bucket": {
"buckets_path": "parent_category>_count"
}
}
}
}
}
}gocode 例九例十代码片断
func (m *metaDataService) CateChartService(ctx context.Context, in *model.CateChartInput) (list map[string][]model.AggItemV, err error) {
ctx, span := gtrace.NewSpan(ctx, "MetaData.CateChartService")
defer span.End()
if len(in.Fields) == 0 {
in.Fields = consts.MetaAggAllFields
}
if in.TopN == 0 {
in.TopN = 10
}
aggFieldArr := strings.Split(in.Fields, ",")
query := elasticCore.NewBoolQuery()
// 生成es查询语句
filters := []*Filters{
&Filters{
Qu: "update_at",
Wildcard: ">=",
T: "now-5y",
},
}
// 组装查询条件
query = buildFilters(ctx, query, filters)
if len(in.CPCate) > 0 {
query = m.buildEsQueryService(ctx, "c_parent_category", in.CPCate)
}
q := elasticCore.NewConstantScoreQuery(query).Boost(1.2)
sourceQ2, _ := q.Source()
fofacore.PrintQuery(sourceQ2)
// 聚合
var csr = make([]model.AggItemV, 0)
var listMap = make(map[string][]model.AggItemV, 0)
for _, field := range aggFieldArr {
switch field {
case "port", "os", "protocol",
"country", "country_name", "region", "city":
csr, err = m.AggWithFieldService(ctx, field, in.TopN, q)
if err != nil {
glog.Errorf(ctx, "m.AggWithFieldService failed, srField:[%v] err:[%v] ", field, err)
}
listMap[field] = csr
case "product", "level", "category", "parent_category", "softhard", "company":
csr, err = m.AggWithFieldNestedService(ctx, field, in.TopN, q)
if err != nil {
glog.Errorf(ctx, "m.AggWithFieldNestedService failed,srField:[%v] err:[%v]", field, err)
}
listMap[field] = csr
default:
}
}
//glog.Info(ctx, "listMap:", listMap)
// 返回结果
return listMap, err
}
// AggItemV ========================================
type AggItemV struct {
Name string `json:"name"`
Value int64 `json:"value"`
Percent float64 `json:"percent"`
}
// 生成es查询语句
func (m *metaDataService) buildEsQueryService(ctx context.Context, serachFields, value string) (esQuery *elasticCore.BoolQuery) {
serachFieldArr := strings.Split(serachFields, ",")
query := elasticCore.NewBoolQuery()
var filters []*Filters
for _, field := range serachFieldArr {
switch field {
case "c_parent_category":
filters = append(filters, &Filters{
Qu: "critical.parent_category",
Wildcard: "",
T: value,
})
case "c_category":
filters = append(filters, &Filters{
Qu: "critical.c_category",
Wildcard: "",
T: value,
})
case "port", "os", "protocol":
filters = append(filters, &Filters{
Qu: field,
Wildcard: "",
T: value,
})
case "country":
filters = append(filters, &Filters{
Qu: "geoip." + field,
Wildcard: "",
T: strings.ToUpper(value),
})
case "country_name", "region", "city":
filters = append(filters, &Filters{
Qu: "geoip." + field,
Wildcard: "",
T: value,
})
case "product", "level", "category", "parent_category", "softhard", "company":
query = query.Must(
elasticCore.NewNestedQuery(
"products",
elasticCore.NewBoolQuery().
Must(
elasticCore.NewTermQuery("products."+value, value),
),
),
)
default:
filters = append(filters, &Filters{
Qu: value,
Wildcard: "",
T: value,
})
}
}
// 组装查询条件
query = buildFilters(ctx, query, filters)
return query
}
type Filters struct {
Qu string // 字段名 如： is_ipv6
Wildcard string //通配符 如： “=” “!=” ">=" "<=" "in"
T interface{} ////搜索值 如： false， 当Wildcard 为 range时 T= "now-5y,now",
}
//AggWithFieldService 指定字段聚合
func (m *metaDataService) AggWithFieldService(ctx context.Context, aggField string, aggFieldSize int, esQuery *elasticCore.ConstantScoreQuery) (csr []model.AggItemV, err error) {
ctx, span := gtrace.NewSpan(ctx, "MetaData.AggWithFieldService")
defer span.End()
// 组装查询条件
// 聚合
var aggFieldNameMap = map[string]string{
"port": "port",
"os": "os",
"protocol": "protocol",
"country": "geoip.country",
"country_name": "geoip.country_name",
"region": "geoip.region",
"city": "geoip.city",
}
aggFieldQ := aggField
if _, ok := aggFieldNameMap[aggField]; ok {
aggFieldQ = aggFieldNameMap[aggField]
}
aggQ := elasticCore.NewTermsAggregation().Field(aggFieldQ).Size(aggFieldSize).OrderByCountDesc()
sourceQ1, _ := aggQ.Source()
fofacore.PrintQuery(sourceQ1)
aggSumQ := elasticCore.NewSumBucketAggregation().BucketsPath(aggField + ">_count")
sourceQ2, _ := aggSumQ.Source()
fofacore.PrintQuery(sourceQ2)
// 查询结果
res, err := EsClient(string(ast.EsMetadata)).
Size(0).
Query(esQuery).
Aggregation(aggField, aggQ).
Aggregation("sum", aggSumQ).
Pretty(true).
Do(context.Background())
if err != nil {
glog.Errorf(ctx, "AggWithFieldService esquery failed: err:[%v]", err)
}
group, _ := res.Aggregations.Terms(aggField)
sumGroup, _ := res.Aggregations.Terms("sum")
var total float64 = 0
for _, bucket := range sumGroup.Aggregations {
marshalJSON, _ := bucket.MarshalJSON()
total, _ = strconv.ParseFloat(string(marshalJSON), 64)
}
if len(group.Aggregations) == 0 {
return
}
var result []model.AggItemV
for _, cList := range group.Buckets {
var resultItem model.AggItemV
if key, ok := cList.Key.(string); ok {
resultItem.Name = key
} else if key, ok := cList.Key.(float64); ok {
resultItem.Name = strconv.Itoa(int(key))
} else if key, ok := cList.Key.(interface{}); ok {
resultItem.Name = convertor.ToString(key)
}
resultItem.Value = cList.DocCount
if proportion, err := strconv.ParseFloat(fmt.Sprintf("%.2f", float64(cList.DocCount)/total*100), 64); err == nil {
resultItem.Percent = proportion
}
result = append(result, resultItem)
}
//glog.Info(ctx, "result:", result)
// 返回结果
return result, err
}
//AggWithFieldNestedService 指定字段聚合Nested属性
func (m *metaDataService) AggWithFieldNestedService(ctx context.Context, aggField string, aggFieldSize int, esQuery *elasticCore.ConstantScoreQuery) (csr []model.AggItemV, err error) {
ctx, span := gtrace.NewSpan(ctx, "MetaData.AggWithFieldNestedService")
defer span.End()
// 组装查询条件
// 聚合
var aggFieldNameMap = map[string]string{
"product": "products.product",
"level": "products.level",
"category": "products.category",
"parent_category": "products.parent_category",
"softhard": "products.softhard",
"company": "products.company",
}
var pathNameMap = map[string]string{
"product": "products",
"level": "products",
"category": "products",
"parent_category": "products",
"softhard": "products",
"company": "products",
}
aggSumQ := elasticCore.NewSumBucketAggregation().BucketsPath(aggField + ">_count")
//sourceQ2, _ := aggSumQ.Source()
//fofacore.PrintQuery(sourceQ2)
categoryAgg := elasticCore.NewTermsAggregation().Field(aggFieldNameMap[aggField]).Size(aggFieldSize).OrderByCountDesc()
nestedAgg := elasticCore.NewNestedAggregation().Path(pathNameMap[aggField]).
SubAggregation(aggField, categoryAgg).
SubAggregation("sum", aggSumQ)
sourceQ1, _ := nestedAgg.Source()
fofacore.PrintQuery(sourceQ1)
// 查询结果
res, err := EsClient(string(ast.EsMetadata)).
Size(0).
Query(esQuery).
Aggregation("group", nestedAgg).
Pretty(true).
Do(context.Background())
if err != nil {
glog.Errorf(ctx, "AggWithFieldNestedService esquery failed: err:[%v]", err)
}
countryGroup, _ := res.Aggregations.Terms("group")
category, _ := countryGroup.Aggregations.Terms(aggField)
sumAgg, _ := countryGroup.Aggregations.Terms("sum")
var total float64 = 0
for _, bucket := range sumAgg.Aggregations {
marshalJSON, _ := bucket.MarshalJSON()
total, err = strconv.ParseFloat(string(marshalJSON), 64)
}
if len(countryGroup.Aggregations) == 0 {
return
}
var result []model.AggItemV
for _, cList := range category.Buckets {
var resultItem model.AggItemV
resultItem.Name = cList.Key.(string)
resultItem.Value = cList.DocCount
if proportion, err := strconv.ParseFloat(fmt.Sprintf("%.2f", float64(cList.DocCount)/total*100), 64); err == nil {
resultItem.Percent = proportion
}
result = append(result, resultItem)
}
//glog.Info(ctx, "result:", result)
// 返回结果
return result, err
}
结果：
{
"took" : 52,
"timed_out" : false,
"_shards" : {
"total" : 6,
"successful" : 6,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 183,
"relation" : "eq"
},
"max_score" : null,
"hits" : [ ]
},
"aggregations" : {
"productsGroup" : {
"doc_count" : 184,
"first_category" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 0,
"buckets" : [
{
"key" : "服务器管理产品",
"doc_count" : 19
},
{
"key" : "其他类型",
"doc_count" : 16
},
{
"key" : "服务器软件",
"doc_count" : 16
},
{
"key" : "行业产品",
"doc_count" : 16
},
{
"key" : "服务器设备",
"doc_count" : 15
},
{
"key" : "Web组件",
"doc_count" : 14
},
{
"key" : "企业应用软件",
"doc_count" : 14
},
{
"key" : "云平台软件",
"doc_count" : 13
},
{
"key" : "设备模块",
"doc_count" : 13
},
{
"key" : "存储设备",
"doc_count" : 11
},
{
"key" : "网络交换设备",
"doc_count" : 11
},
{
"key" : "物联网设备",
"doc_count" : 10
},
{
"key" : "网络安全产品",
"doc_count" : 9
},
{
"key" : "办公外设",
"doc_count" : 7
}
]
},
"sum" : {
"value" : 184.0
}
}
}
}例十一 查不为null 或者 为null 的数据
# vulnerabilities字段(nested属性数组的) 不为null的或者空的
GET metadata/_search
{
"query": {
"nested": {
"path": "vulnerabilities",
"query": {
"bool": {
"must": {
"exists": {
"field": "vulnerabilities"
}
}
}
}
}
}
}
# vulnerabilities字段(nested属性数组的) 为null的 或者空的
POST /metadata/_doc/_search
{
"query": {
"bool": {
"must_not": [
{
"exists": {
"field": "vulnerabilities"
}
}
]
}
}
}
# critical字段（对像类型字段） 不为null的
GET metadata/_search
{
"query": {
"bool": {
"must": [
{
"exists": {
"field": "critical"
}
}
]
}
}
}
# critical字段（对像类型字段） 为null的
GET metadata/_search
{
"query": {
"bool": {
"must_not": [
{
"exists": {
"field": "critical"
}
}
]
}
}
}
# 需要查询出critical 不为 null 并且 critical.parent_category不为 "" 的数据
GET metadata/_search
{
"_source": [
"critical"
],
"query": {
"bool": {
"must": [
{
"exists": {
"field": "critical"
}
},{
"bool": {
"must_not": [
{
"term": {
"critical.parent_category": {
"value": ""
}
}
}
]
}
}
]
}
}
}
# 现需要查询出tags为 "" 或者为 null 的数据
{
"query": {
"bool": {
"must": {
"match_all": {}
},
"filter": [
{
"terms": {
"_id": [
"ec5db526a7b32bdd1f6ac865cf830436",
"e0422a0641334293810275a5302b5014"
]
}
},
{
"bool": {
"should": [
{
"term": {
"tags": ""
}
},
{
"bool": {
"must_not": [
{
"exists": {
"field": "tags"
}
}
]
}
}
]
}
}
]
}
},
"size": 100,
"sort": [],
"_source": [
"uuid",
"tags"
]
}inner_hits{} 查出products.softhard为2的数据
# inner_hits{} 查出products.softhard为2的数据
GET metadata/_search
{
"_source": [
"products",
"ip",
"port"
],
"size": 10,
"query": {
"constant_score": {
"filter": {
"bool": {
"must": [
{
"range": {
"update_at": {
"from": "now-5y",
"include_lower": true,
"include_upper": true,
"to": null
}
}
},
{
"term": {
"ip": "10.10.11.12"
}
},
{
"term": {
"port": "81"
}
},
{
"nested": {
"path": "products",
"query": {
"bool": {
"must": [
{
"term": {
"products.softhard": {
"value": "2"
}
}
}
]
}
},
"inner_hits": {}
}
}
]
}
},
"boost": 1.2
}
}
}返回数据中多了inner_hits字段，就是筛选出来的结果
{
"took" : 2,
"timed_out" : false,
"_shards" : {
"total" : 6,
"successful" : 6,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 1,
"relation" : "eq"
},
"max_score" : 1.2,
"hits" : [
{
"_index" : "metadata",
"_type" : "_doc",
"_id" : "10.10.11.12:81",
"_score" : 1.2,
"_source" : {
"port" : 81,
"ip" : "10.10.11.12",
"products" : [
{
"product" : "授权系统",
"level" : 0,
"parent_category" : "网络产品",
"softhard" : "2",
"company" : "Huawei_rj",
"category" : "ADSL",
"version" : ""
},
{
"product" : "路由器AX6 ",
"level" : 1,
"parent_category" : "网络产品",
"softhard" : "1",
"company" : "Huawei_yj",
"category" : "ADSL",
"version" : ""
},
{
"product" : "路由器AX7 ",
"level" : 1,
"parent_category" : "网络产品",
"softhard" : "1",
"company" : "Huawei_yj",
"category" : "ADSL",
"version" : ""
}
]
},
"inner_hits" : {
"products" : {
"hits" : {
"total" : {
"value" : 1,
"relation" : "eq"
},
"max_score" : 7.9944715,
"hits" : [
{
"_index" : "metadata",
"_type" : "_doc",
"_id" : "10.10.11.12:81",
"_nested" : {
"field" : "products",
"offset" : 0
},
"_score" : 7.9944715,
"_source" : {
"product" : "授权系统",
"level" : 0,
"parent_category" : "网络产品",
"softhard" : "2",
"company" : "Huawei_rj",
"category" : "ADSL",
"version" : ""
}
}
]
}
}
}
}
]
}
}聚合出products.softhard 为1的数据
# inner_hits https://www.jianshu.com/p/0d6488a8072b
# 聚合出products.softhard 为1的数据
GET metadata/_search
{
"_source": [
"products",
"ip",
"port"
],
"size": 2,
"query": {
"constant_score": {
"filter": {
"bool": {
"must": [
{
"range": {
"update_at": {
"from": "now-5y",
"include_lower": true,
"include_upper": true,
"to": null
}
}
},
{
"term": {
"ip": "10.10.11.12"
}
},
{
"term": {
"port": "81"
}
}
]
}
},
"boost": 1.2
}
},
"aggs": {
"companyGroup": {
"aggs": {
"filter_softhard": {
"aggs": {
"company": {
"terms": {
"field": "products.company",
"order": [
{
"_count": "desc"
}
],
"size": 10
}
}
},
"filter": {
"term": {
"products.softhard": 1
}
}
}
},
"nested": {
"path": "products"
}
}
}
}
返回结果：
{
"took" : 2,
"timed_out" : false,
"_shards" : {
"total" : 6,
"successful" : 6,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 1,
"relation" : "eq"
},
"max_score" : 1.2,
"hits" : [
{
"_index" : "metadata",
"_type" : "_doc",
"_id" : "10.10.11.12:81",
"_score" : 1.2,
"_source" : {
"port" : 81,
"ip" : "10.10.11.12",
"products" : [
{
"product" : "授权系统",
"level" : 0,
"parent_category" : "网络产品",
"softhard" : "2",
"company" : "Huawei_rj",
"category" : "ADSL",
"version" : ""
},
{
"product" : "路由器AX6 ",
"level" : 1,
"parent_category" : "网络产品",
"softhard" : "1",
"company" : "Huawei_yj",
"category" : "ADSL",
"version" : ""
},
{
"product" : "路由器AX7 ",
"level" : 1,
"parent_category" : "网络产品",
"softhard" : "1",
"company" : "Huawei_yj",
"category" : "ADSL",
"version" : ""
}
]
}
}
]
},
"aggregations" : {
"companyGroup" : {
"doc_count" : 3,
"filter_softhard" : {
"doc_count" : 2,
"company" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 0,
"buckets" : [
{
"key" : "Huawei_yj",
"doc_count" : 2
}
]
}
}
}
}
}
过滤漏洞 “vulnerabilities.vul_name”=””matrix”或”feed” ，并求分桶之合
##过滤漏洞 "vulnerabilities.vul_name"=""matrix"或"feed" ，并求分桶之合
GET metadata/_search
{
"_source": [
"products",
"ip",
"port",
"vulnerabilities"
],
"size": 10,
"query": {
"constant_score": {
"filter": {
"bool": {
"must": [
{
"range": {
"update_at": {
"from": "now-5y",
"include_lower": true,
"include_upper": true,
"to": null
}
}
},
{
"term": {
"ip": "ae96:c471:5498:2c53:b019:a53d:ff02:8cbb"
}
},
{
"term": {
"port": "52131"
}
}
]
}
},
"boost": 1.2
}
},
"aggs": {
"vulnerabilitiesGroup": {
"nested": {
"path": "vulnerabilities"
},
"aggs": {
"vul_name_filter": {
"filter": {
"terms": {
"vulnerabilities.vul_name": ["matrix","feed"]
}
},
"aggs": {
"vul_name": {
"terms": {
"field": "vulnerabilities.vul_name",
"size": 25
}
},
"sum": {
"sum_bucket": {
"buckets_path": "vul_name>_count"
}
}
}
}
}
}
}
}返参
{
"took" : 3,
"timed_out" : false,
"_shards" : {
"total" : 6,
"successful" : 6,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 1,
"relation" : "eq"
},
"max_score" : 1.2,
"hits" : [
{
"_index" : "metadata",
"_type" : "_doc",
"_id" : "ae96:c471:5498:2c53:b019:a53d:ff02:8cbb:52131",
"_score" : 1.2,
"_source" : {
"port" : 52131,
"ip" : "ae96:c471:5498:2c53:b019:a53d:ff02:8cbb",
"vulnerabilities" : [
{
"vul_name" : "feed",
"third_id" : "",
"vul_level" : 3,
"vul_id" : "CNVD.1679481666805",
"vul_scan_at" : "2023-03-22 17:41:06",
"cve_id" : "",
"vul_url" : "http://www.seniormindshare.net/niches/enable/content/relationships",
"cnvd_id" : "1679481666805",
"vul_type" : "optical",
"cnnvd_id" : ""
},
{
"vul_name" : "matrix",
"third_id" : "1679481666805",
"vul_level" : 3,
"vul_id" : "Third.1679481666805",
"vul_scan_at" : "2023-03-22 17:41:06",
"cve_id" : "",
"vul_url" : "http://www.legacyapplications.biz/real-time",
"cnvd_id" : "",
"vul_type" : "multi-byte",
"cnnvd_id" : ""
},
{
"vul_name" : "matrix",
"third_id" : "",
"vul_level" : 4,
"vul_id" : "CVE.1679481666805",
"vul_scan_at" : "2023-03-22 17:41:06",
"cve_id" : "1679481666805",
"vul_url" : "https://www.forwardintuitive.io/dot-com/granular/efficient",
"cnvd_id" : "",
"vul_type" : "solid state",
"cnnvd_id" : ""
},
{
"vul_name" : "application",
"third_id" : "",
"vul_level" : 2,
"vul_id" : "CNVD.1679481666805",
"vul_scan_at" : "2023-03-22 17:41:06",
"cve_id" : "",
"vul_url" : "https://www.customergenerate.biz/engage",
"cnvd_id" : "1679481666805",
"vul_type" : "haptic",
"cnnvd_id" : ""
},
{
"vul_name" : "bandwidth",
"third_id" : "",
"vul_level" : 3,
"vul_id" : "CNNVD.1679481666805",
"vul_scan_at" : "2023-03-22 17:41:06",
"cve_id" : "",
"vul_url" : "http://www.internationalcustomized.net/portals",
"cnvd_id" : "",
"vul_type" : "auxiliary",
"cnnvd_id" : "1679481666805"
}
],
"products" : [
{
"product" : "Combwill",
"level" : 10,
"parent_category" : "服务器设备",
"softhard" : "1",
"company" : "OptumInsight",
"category" : "串口服务器",
"version" : "4.7.7"
}
]
}
}
]
},
"aggregations" : {
"vulnerabilitiesGroup" : {
"doc_count" : 5,
"vul_name_filter" : {
"doc_count" : 3,
"vul_name" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 0,
"buckets" : [
{
"key" : "matrix",
"doc_count" : 2
},
{
"key" : "feed",
"doc_count" : 1
}
]
},
"sum" : {
"value" : 3.0
}
}
}
}
}过滤一级数据为空的数据 聚合parent_category下的second_category、company、product
# 过滤一级数据为空的数据 聚合parent_category下的second_category、company、product
GET metadata/_search
{
"size": 1,
"query": {
"constant_score": {
"filter": {
"bool": {
"must": [
{
"range": {
"update_at": {
"from": "now-5y",
"include_lower": true,
"include_upper": true,
"to": null
}
}
},
{
"term": {
"geoip.country": "CN"
}
},
{
"nested": {
"path": "products",
"query": {
"bool": {
"must_not": [
{
"term": {
"products.parent_category": ""
}
}
]
}
}
}
}
]
}
},
"boost": 1.2
}
},
"aggs": {
"group": {
"aggregations": {
"filterAgg": {
"aggregations": {
"first_category": {
"aggregations": {
"company": {
"terms": {
"field": "products.company",
"order": [
{
"_count": "desc"
}
],
"size": 50
}
},
"product": {
"terms": {
"field": "products.product",
"order": [
{
"_count": "desc"
}
],
"size": 50
}
},
"second_category": {
"terms": {
"field": "products.category",
"order": [
{
"_count": "desc"
}
],
"size": 50
}
}
},
"terms": {
"field": "products.parent_category",
"order": [
{
"_count": "desc"
}
],
"size": 50
}
}
},
"filter": {
"bool": {
"must_not": {
"term": {
"products.parent_category": ""
}
}
}
}
}
},
"nested": {
"path": "products"
}
}
}
}返回
{
"took" : 8,
"timed_out" : false,
"_shards" : {
"total" : 6,
"successful" : 6,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 1029,
"relation" : "eq"
},
"max_score" : 1.2,
"hits" : [
{
"_index" : "metadata",
"_type" : "_doc",
"_id" : "124.63.255.17:6501",
"_score" : 1.2,
"_source" : {
"id" : "124.63.255.17:6501",
"ip" : "124.63.255.17",
"create_at" : "2023-03-17 15:52:26",
"update_at" : "2023-03-17 15:52:26",
"type" : "metadata",
"port" : 6500,
"protocol" : "ssh",
"base_protocol" : "",
"geoip" : {
"isp" : "Fuzion Apps, Inc.",
"city" : "北京市",
"region" : "北京",
"country" : "CN",
"country_name" : "中国",
"continent" : "Oceania",
"timezone" : "Romance Standard Time",
"longitude" : 113.965229,
"latitude" : -85.548254
},
"host" : "humansynergies.io",
"hostnames" : [
"corporateengineer.info"
],
"domain" : [ ],
"icp" : "",
"subdomain" : [ ],
"os" : [
"Pinkposse",
"leadvortals.com"
],
"server" : "nginx 2.2.10",
"title" : "个人银行登录系统 22",
"header" : "",
"banner" : "",
"body" : "",
"cert" : "",
"certs" : {
"cert_date" : "1921-03-31 02:32:29",
"cert_num" : 25100,
"domain" : "centralcontent.io",
"is_valid" : false,
"issuer_cn" : "Navico",
"issuer_cns" : [
"Propeller Health"
],
"issuer_org" : "PowerAdvocate",
"not_after" : "1944-05-15 09:58:19",
"not_before" : "1990-07-24 02:31:48",
"sig_alth" : "787900487",
"sn" : "260224897",
"subject_cn" : "PIXIA Corp",
"subject_cns" : [
"NextBus"
],
"subject_org" : "Russell Investments",
"valid_type" : "letterpress"
},
"critical" : {
"parent_category" : "金融",
"category" : "保险"
},
"asn" : {
"number" : "12580",
"organization" : "China Mobile"
},
"products" : [
{
"product" : "Mysql",
"level" : 2,
"category" : "数据库",
"parent_category" : "系统软件",
"softhard" : "2",
"company" : "Mysql",
"version" : "1.14.9"
},
{
"product" : "nginx",
"level" : 2,
"category" : "服务代理",
"parent_category" : "支撑系统",
"softhard" : "2",
"company" : "nginx",
"version" : "1.14.9"
}
],
"dbs" : { },
"favicon" : {
"base64" : "The HDD matrix is down, back up the auxiliary microchip so we can verify the XSS bus!",
"hash" : "The GB system is down, validate the solid state protocol so we can deconstruct the SSL sensor!",
"url" : "https://www.regionalend-to-end.org/visionary/monetize/intuitive/e-enable"
},
"vulnerabilities" : [ ]
}
}
]
},
"aggregations" : {
"group" : {
"doc_count" : 1051,
"filterAgg" : {
"doc_count" : 1046,
"first_category" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 43,
"buckets" : [
{
"key" : "网络产品",
"doc_count" : 1003,
"product" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 17,
"buckets" : [
{
"key" : "NGINX,Baidu-站长平台,php",
"doc_count" : 986
}
]
},
"second_category" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 17,
"buckets" : [
{
"key" : "服务器",
"doc_count" : 986
}
]
},
"company" : {
"doc_count_error_upper_bound" : 0,
"sum_other_doc_count" : 17,
"buckets" : [
{
"key" : "华顺",
"doc_count" : 986
}
]
}
}
]
}
}
}
}
}
作者：海马 创建时间：2023-02-15 23:33
最后编辑：海马 更新时间：2024-05-10 15:32
最后编辑：海马 更新时间：2024-05-10 15:32
