创建一个错误

package main

import (
    "errors"
    "fmt"
)

func main() {
    err := fmt.Errorf("123123")
    fmt.Println("err1:", err)  //err1: 123123

    err = errors.New("math: square root of negative number")
    fmt.Println("err:2", err) //err:2 math: square root of negative number

    name := getError("name no is empty.")
    fmt.Println(name) //I am err massage.name no is empty.
}

func getError(str string) error{
 return errors.New("I am err massage."+str)
}

数组分批处理

func GKDataImport() {
    //t := `2021-09-27 22:18:32`
    //t = strings.Replace(t, " ", "_", -1)
    //fmt.Println(t)
    //return
    var filePath string
    //filePath = "log/jsonTagData/company.json"
    filePath = config.CONFIG.EsClient.SourceFilePath
    list := make([]saveGongKongData, 0, 1000)
    file, err := os.Open(filePath)
    if err != nil {
        fmt.Println("GKDataImport os.Open(filePath) err:", err)
        return
    }
    defer file.Close()

    scanner := bufio.NewReader(file)
    var errDataIds []string
    // total 总共读取的数据条数
    // nTotal 处理数据总条数
    // n 当前一轮多少个，每次批量写入时清0
    total, nTotal, n, eof := 0, 0, 0, false
    for {
        //line, _, err := scanner.ReadLine()
        line, err := scanner.ReadBytes('\n')
        if err != nil {
            if err == io.EOF {
                eof = true
            } else {
                fmt.Println(err.Error())
                break
            }
        }
        total++

        if len(line) == 0 {
            //errMsg := list[len(list)-1].ID + "下一行错误,数据为空"
            errMsg := fmt.Sprintf("数据为空:%s  line:%d err:%v \n", string(line), total, err)
            utils.LogDebugWithPath("log", "GKDataImport", errMsg+":"+string(line))
            errDataIds = append(errDataIds, errMsg)
        } else if len(line) != 0 && json.Valid(line) {
            var ds = saveGongKongData{}
            err = json.Unmarshal(line, &ds)
            if err != nil {
                errMsg := list[len(list)-1].ID + "下一行错误，json.Unmarshal fail err: " + err.Error()
                errDataIds = append(errDataIds, errMsg)
                errMsgSave := fmt.Sprintf("import data file: %s failed, err: %s \n", path.Base(filePath), err.Error())
                log.Println(errMsgSave)
                utils.LogDebugWithPath("log", "GKDataImport", errMsgSave)
                break
            }
            list = append(list, ds)
        } else {
            errMsg := fmt.Sprintf("数据其它错误:%s  line:%d err:%v \n", string(line), total, err)
            utils.LogDebugWithPath("log", "GKDataImport", errMsg+":"+string(line))
            errDataIds = append(errDataIds, errMsg)
        }

        n++
        if n == 1000 || eof {
            nTotal += len(list)
            log.Printf("已经读取 %d 条, 已处理处理 %d 条数据", total, nTotal)
            n = 0
            if err = CreateImportByBulk(&list); err != nil {
                break
            }
            if eof {
                break
            }
            list = list[0:0]
        }
    }

    if err != nil {
        fmt.Printf("import dns file: %s failed, err: %s", path.Base(filePath), err.Error())
        return
    }
    log.Println("===============================================>")
    msg := fmt.Sprintf("共计 %d 条数据  处理完成 %d 条数据 ,错误 %d 条 errDataIds:%v file:%s \n", total, nTotal, total-nTotal, errDataIds, filePath)
    log.Println(msg)
    utils.LogDebugWithPath("log/exportEs", "exportEs"+config.CONFIG.EsClient.IndexName, msg)
    return
}


func CreateImportByBulk(list *[]saveGongKongData) error {

    req := es.EsClient.Bulk().Index(config.CONFIG.EsClient.IndexName).
        Type("service") //es6需要指定tpye  es7 不需要
    for _, item := range *list {
        doc := elastic.NewBulkIndexRequest().
            Id(item.ID).
            Doc(item.Source)
        req.Add(doc)
    }
    actions := req.NumberOfActions()
    if actions <= 0 {
        return fmt.Errorf("req.NumberOfActions < 0 ,list:%v", list)
    }
    bResp, err := req.Refresh("true").Do(context.Background())
    if err != nil {
        log.Printf("CreateImportByBulk fail err:%v", err)
    }
    var success = make([]string, 0)
    type failedDetail struct {
        Ids         []string
        ErrorMagmap map[string]string
    }
    var fDetail failedDetail
    // 统计结果
    for _, v := range bResp.Items {
        var item = *v["index"]
        //if v["index"].Status == 201 {
        //    fmt.Printf("k--->:%#v , status:%#v  err:%v  v:%#v \n ", k, v["index"].Status, item.Error, v["index"])
        //}
        if item.Error == nil {
            success = append(success, item.Id)
        } else {
            fDetail.Ids = append(fDetail.Ids, item.Id)
            b, _ := json.Marshal(item.Error)
            fDetail.ErrorMagmap = make(map[string]string, 0)
            fDetail.ErrorMagmap[item.Id] = string(b)
        }
    }
    //responseMsg := fmt.Sprintf("处理条数: %d 条 , 成功: %d 条, 失败 %d 条.", len(*list), len(success), len(fDetail.Ids))
    //fmt.Println("responseMsg:", responseMsg)
    return err
}

json字符串转为map

content = `
{"openDns":1,"taskType":2,"isWait":1,"copyTargets":"static/downloads/1.xlsx","targetGroups":null,"targets":["static/downloads/1.xlsx"],"timeout":30,"domain":true,"subDomain":false,"ipicp":false,"whois":false,"probeTarget":1,"icpQueryMaxNum":100,"nodeIDList":"","dnsServer":"114.114.114.114","filePath":"static/downloads/1.xlsx"}
`
mapData := make(map[string]interface{})
if err = json.Unmarshal(content, &mapData); err != nil {
        return
}

清空切片

package main

import "fmt"

func main() {
    var names = []string{"lisi", "wangwu", "zhangsan"}
    fmt.Println("name1:", names) //name1: [lisi wangwu zhangsan]
    names = RemoveIndex(names, 0)
    fmt.Println("name2:", names) //name2: [wangwu zhangsan]

    //通过输出，我们很容易得到如下结论：
    //通过切片表达式清空切片，仅长度归0，而容量维持不变
    //解决了可能扩容的问题
    //清空后，切片指向的底层数组也不变
    //解决了更换底层数组，开辟新空间，以及可能的垃圾回收问题
    //注意：切片指向的底层数组不变，也就导致了无论是通过切片操作还是数组操作，都会影响彼此。
    //如果仅仅是清空切片，后续继续 append 操作，博主更推荐切片表达式的方法[0:0] 。
    // 原文:https://blog.csdn.net/m0_70556273/article/details/127077579
    names = names[0:0]
    fmt.Println("name3:", names) //name3: []

}
func RemoveIndex(s []string, index int) []string {
    return append(s[:index], s[index+1:]...)
}

Golang 切片/数组实现分页

func SlicePage(page, pageSize, nums int64) (sliceStart, sliceEnd int64) {
    if page <= 0 {
        page = 1
    }
    if pageSize < 0 {
        pageSize = 20 //设置一页默认显示的记录数
    }
    if pageSize > nums {
        return 0, nums
    }
    // 总页数
    pageCount := int64(math.Ceil(float64(nums) / float64(pageSize)))
    if page > pageCount {
        return 0, 0
    }
    sliceStart = (page - 1) * pageSize
    sliceEnd = sliceStart + pageSize

    if sliceEnd > nums {
        sliceEnd = nums
    }
    return sliceStart, sliceEnd
}

测试：

func main() {
    var data = []string{"qww", "dsassa", "1313", "jasdd", "dsawdqw"}
    start, end := SlicePage(1, 3, int64(len(data))) //第一页1页显示3条数据
    afterLimitData := data[start:end]               //分页后的数据
    fmt.Println("分页后的数据:", afterLimitData)
    fmt.Println("数据总量:", len(data))
}

结果

 分页后的数据: ["qww", "dsassa", "1313"]
 数据总量:5

// 去前缀
func FormatResult(res []byte) (string, error) {
    trimKeyPrefix, err := TrimKeyPrefixToString(string(res), consts.ColPrefix, []string{})
    if err != nil {
        return "", err
    }
    jsonStrTmp, err := gjson.DecodeToJson(trimKeyPrefix)
    if err != nil {
        return "", err
    }
    if jsonStrTmp.Contains("hits") {
        return jsonStrTmp.Get("hits.hits").String(), nil

    }
    return jsonStrTmp.MustToJsonString(), nil
}

// TrimKeyPrefixToMap 给 map 去除 key 前缀
// dataStr    数据源 map
// prefix  要删除的前缀
// ignores 排除的字段
func TrimKeyPrefixToString(dataStr, prefix string, ignores []string) (string, error) {
    if len(prefix) == 0 {
        return dataStr, nil
    }
    var dataMap = make(map[string]interface{})
    var targetMap = make(map[string]interface{})

    err := json.Unmarshal([]byte(dataStr), &dataMap)
    if err != nil {
        return "", err
    }
    toMap := TrimKeyPrefixToMap(dataMap, targetMap, prefix, ignores...)
    marshal, err := json.Marshal(toMap)
    if err != nil {
        return "", err
    }
    return string(marshal), nil
}




// AddKeyPrefixToMap 给 map 添加 key 前缀
// data    数据源 map
// target  目标   map
// prefix  要添加的前缀
// ignores 排除的字段
func AddKeyPrefixToMap(data, target map[string]interface{}, prefix string, ignores ...string) map[string]interface{} {
    passMap := make(map[string]struct{})
    for _, ig := range ignores {
        passMap[ig] = struct{}{}
    }
    for k, m := range data {
        if m == nil {
            continue
        }
        _, pass := passMap[k]
        mType := reflect.TypeOf(m)
        mValue := reflect.ValueOf(m)
        var d interface{}
        if mType.Kind() == reflect.Map {
            m1 := m.(map[string]interface{})
            var subTarget = make(map[string]interface{})
            subTarget = AddKeyPrefixToMap(m1, subTarget, prefix, ignores...)
            d = subTarget
        } else if mType.Kind() == reflect.Slice {
            l := mValue.Len()
            arr := make([]interface{}, 0, l)
            for i := 0; i < l; i++ {
                value := mValue.Index(i) // Value of item
                itemType := value.Type() // Type of item
                switch itemType.Kind() {
                case reflect.Map:
                    // 递归加前缀
                    m2 := value.Interface().(map[string]interface{})
                    var subTarget = make(map[string]interface{})
                    subTarget = AddKeyPrefixToMap(m2, subTarget, prefix, ignores...)
                    arr = append(arr, subTarget)
                case reflect.Interface:
                    val := value.Interface()
                    if m2, ok := val.(map[string]interface{}); ok {
                        var subTarget = make(map[string]interface{})
                        subTarget = AddKeyPrefixToMap(m2, subTarget, prefix, ignores...)
                        arr = append(arr, subTarget)
                    } else {
                        arr = append(arr, val)
                    }
                case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
                    arr = append(arr, value.Int())
                case reflect.String:
                    arr = append(arr, value.String())
                case reflect.Bool:
                    arr = append(arr, value.Bool())
                case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
                    arr = append(arr, value.Uint())
                default:
                    continue
                }
            }
            d = arr
        } else {
            d = m
        }
        if pass {
            target[k] = d
        } else {
            target[prefix+k] = d
        }
    }
    return target
}

// TrimKeyPrefixToMap 给 map 去除 key 前缀
// data    数据源 map
// target  目标   map
// prefix  要删除的前缀
// ignores 排除的字段
func TrimKeyPrefixToMap(data, target map[string]interface{}, prefix string, ignores ...string) map[string]interface{} {
    passMap := make(map[string]struct{})
    for _, ig := range ignores {
        passMap[ig] = struct{}{}
    }
    for k, m := range data {
        if m == nil {
            continue
        }
        _, pass := passMap[k]
        mType := reflect.TypeOf(m)
        mValue := reflect.ValueOf(m)
        var d interface{}
        if mType.Kind() == reflect.Map {
            m1 := m.(map[string]interface{})
            var subTarget = make(map[string]interface{})
            subTarget = TrimKeyPrefixToMap(m1, subTarget, prefix, ignores...)
            d = subTarget
        } else if mType.Kind() == reflect.Slice {
            l := mValue.Len()
            arr := make([]interface{}, 0, l)
            for i := 0; i < l; i++ {
                value := mValue.Index(i) // Value of item
                itemType := value.Type() // Type of item
                switch itemType.Kind() {
                case reflect.Map:
                    // 递归去前缀
                    m2 := value.Interface().(map[string]interface{})
                    var subTarget = make(map[string]interface{})
                    subTarget = TrimKeyPrefixToMap(m2, subTarget, prefix, ignores...)
                    arr = append(arr, subTarget)
                case reflect.Interface:
                    val := value.Interface()
                    if m2, ok := val.(map[string]interface{}); ok {
                        var subTarget = make(map[string]interface{})
                        subTarget = TrimKeyPrefixToMap(m2, subTarget, prefix, ignores...)
                        arr = append(arr, subTarget)
                    } else {
                        arr = append(arr, val)
                    }
                case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
                    arr = append(arr, value.Int())
                case reflect.String:
                    arr = append(arr, value.String())
                case reflect.Bool:
                    arr = append(arr, value.Bool())
                case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
                    arr = append(arr, value.Uint())
                default:
                    continue
                }
            }
            d = arr
        } else {
            d = m
        }
        if pass {
            target[k] = d
        } else {
            target[strings.TrimPrefix(k, prefix)] = d
        }
    }
    return target
}

输出结果：

func TestFormatResult(t *testing.T) {
    //type args struct {
    //    res []byte
    //}

    type args struct {
        dataStr []byte
        prefix  string
        ignores []string
    }
    dataStrData := `{"_id":"https://www.badgerfarmacolbridgeclub.co.uk","_index":"wa_zdzc_db_t_fofapro_subdomain_cert","_source":{"c_favicon":{"base64":"R0lGODdhlgCWAOcAAAQCBCyCLJ6EWs/LIYsGDJyNcZzGlNrCkqKkk0FCGd7iw5xEENCklTR+NN/Jvezy08yEeEQIBOhIQM20h1REKKalniMkH/kCBO7RvvXlyFpXNqKUdNzVsmRiGGRkXOIkItbYxLK0pOu0nodrN8RwZHN0aOjm4M2Vh3zKfHyCFLRi","c_url":"https://www.badgerfarmacolbridgeclub.co.uk/wp-content/uploads/2019/05/Logocards-150x150.gif"},"c_lastupdatetime":"2023-09-26 09:47:03"},"_type":"_doc","sort":[1695721623000]}`
    wantDataStrData := `{"_id":"https://www.badgerfarmacolbridgeclub.co.uk","_index":"wa_zdzc_db_t_fofapro_subdomain_cert","_source":{"favicon":{"base64":"R0lGODdhlgCWAOcAAAQCBCyCLJ6EWs/LIYsGDJyNcZzGlNrCkqKkk0FCGd7iw5xEENCklTR+NN/Jvezy08yEeEQIBOhIQM20h1REKKalniMkH/kCBO7RvvXlyFpXNqKUdNzVsmRiGGRkXOIkItbYxLK0pOu0nodrN8RwZHN0aOjm4M2Vh3zKfHyCFLRi","url":"https://www.badgerfarmacolbridgeclub.co.uk/wp-content/uploads/2019/05/Logocards-150x150.gif"},"lastupdatetime":"2023-09-26 09:47:03"},"_type":"_doc","sort":[1695721623000]}`

    tests := []struct {
        name    string
        args    args
        want    string
        wantErr assert.ErrorAssertionFunc
    }{
        // TODO: Add test cases.
        {
            name: "TestTrimKeyPrefixToString",
            args: args{
                dataStr: []byte(dataStrData),
                prefix:  "c_",
                ignores: []string{},
            },
            want: wantDataStrData,
        },
    }
    for _, tt := range tests {
        t.Run(tt.name, func(t *testing.T) {
            got, err := FormatResult(tt.args.dataStr)
            //if !tt.wantErr(t, err, fmt.Sprintf("FormatResult(%v)", tt.want)) {
            //    return
            //}
            assert.NoError(t, err)
            //assert.Contains(t, res, tt.want)
            assert.Equalf(t, tt.want, got, "FormatResult(%v)", tt.want)
        })
    }
}

数组分组

// []string{"1", "2", "3", "4", "5", "6"} -> 如果N=2,则返回   [][]string{{"1", "2", "3"}, {"4", "5", "6"}}
// SplitIntoGroups 根据指定的大小拆分切片
func SplitIntoGroups[T any](arr []T, N int) [][]T {
    var result [][]T
    for i := 0; i < len(arr); i += N {
        end := i + N
        if end > len(arr) {
            end = len(arr)
        }
        result = append(result, arr[i:end])
    }
    return result
}