From a0fc23dd6480b670c1cb0fb615c3527d67d0c382 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Thu, 18 Jul 2024 11:56:04 +0800 Subject: [PATCH] Generate new rule-sets --- .github/release-rule-set.sh | 25 +++++++++++------ go.mod | 20 ++++++------- go.sum | 56 ++++++++++++++++++++++++------------- main.go | 47 ++++++++++++++++++++++++------- 4 files changed, 99 insertions(+), 49 deletions(-) diff --git a/.github/release-rule-set.sh b/.github/release-rule-set.sh index efb577d7da..1acf21e71a 100755 --- a/.github/release-rule-set.sh +++ b/.github/release-rule-set.sh @@ -2,12 +2,19 @@ set -e -o pipefail -cd rule-set -git init -git config --local user.email "github-action@users.noreply.github.com" -git config --local user.name "GitHub Action" -git remote add origin https://github-action:$GITHUB_TOKEN@github.com/SagerNet/sing-geosite.git -git branch -M rule-set -git add . -git commit -m "Update rule-set" -git push -f origin rule-set +function releaseRuleSet() { + dirName=$1 + pushd $dirName + git init + git config --local user.email "github-action@users.noreply.github.com" + git config --local user.name "GitHub Action" + git remote add origin https://github-action:$GITHUB_TOKEN@github.com/SagerNet/sing-geosite.git + git branch -M $dirName + git add . + git commit -m "Update rule-set" + git push -f origin $dirName + popd +} + +releaseRuleSet rule-set +releaseRuleSet rule-set-unstable diff --git a/go.mod b/go.mod index 5288944b9a..a9624d2d3b 100644 --- a/go.mod +++ b/go.mod @@ -6,10 +6,10 @@ toolchain go1.22.3 require ( github.com/google/go-github/v45 v45.2.0 - github.com/sagernet/sing v0.3.8 - github.com/sagernet/sing-box v1.8.13 + github.com/sagernet/sing v0.5.0-alpha.12.0.20240717075530-332e47007567 + github.com/sagernet/sing-box v1.9.4-0.20240718050851-e2b52d9664a6 github.com/v2fly/v2ray-core/v5 v5.16.1 - google.golang.org/protobuf v1.34.0 + google.golang.org/protobuf v1.34.2 ) require ( @@ -17,13 +17,13 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/logrusorgru/aurora v2.0.3+incompatible // indirect - github.com/miekg/dns v1.1.59 // indirect - github.com/sagernet/sing-dns v0.1.14 // indirect + github.com/miekg/dns v1.1.61 // indirect + github.com/sagernet/sing-dns v0.3.0-beta.10 // indirect go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect - golang.org/x/crypto v0.22.0 // indirect - golang.org/x/mod v0.17.0 // indirect - golang.org/x/net v0.24.0 // indirect + golang.org/x/crypto v0.24.0 // indirect + golang.org/x/mod v0.18.0 // indirect + golang.org/x/net v0.26.0 // indirect golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.19.0 // indirect - golang.org/x/tools v0.20.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/tools v0.22.0 // indirect ) diff --git a/go.sum b/go.sum index 30be19b510..3568da4212 100644 --- a/go.sum +++ b/go.sum @@ -3,6 +3,8 @@ github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -12,18 +14,28 @@ github.com/google/go-github/v45 v45.2.0 h1:5oRLszbrkvxDDqBCNj2hjDZMKmvexaZ1xw/FC github.com/google/go-github/v45 v45.2.0/go.mod h1:FObaZJEDSTa/WGCzZ2Z3eoCDXWJKMenWWTrd8jrta28= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/google/pprof v0.0.0-20231101202521-4ca4178f5c7a h1:fEBsGL/sjAuJrgah5XqmmYsTLzJp/TO9Lhy39gkverk= +github.com/google/pprof v0.0.0-20231101202521-4ca4178f5c7a/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= -github.com/miekg/dns v1.1.59 h1:C9EXc/UToRwKLhK5wKU/I4QVsBUc8kE6MkHBkeypWZs= -github.com/miekg/dns v1.1.59/go.mod h1:nZpewl5p6IvctfgrckopVx2OlSEHPRO/U4SYkRklrEk= +github.com/miekg/dns v1.1.61 h1:nLxbwF3XxhwVSm8g9Dghm9MHPaUZuqhPiGL+675ZmEs= +github.com/miekg/dns v1.1.61/go.mod h1:mnAarhS3nWaW+NVP2wTkYVIZyHNJ098SJZUki3eykwQ= +github.com/onsi/ginkgo/v2 v2.10.0 h1:sfUl4qgLdvkChZrWCYndY2EAu9BRIw1YphNAzy1VNWs= +github.com/onsi/ginkgo/v2 v2.10.0/go.mod h1:UDQOh5wbQUlMnkLfVaIUMtQ1Vus92oM+P2JX1aulgcE= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/sagernet/sing v0.3.8 h1:gm4JKalPhydMYX2zFOTnnd4TXtM/16WFRqSjMepYQQk= -github.com/sagernet/sing v0.3.8/go.mod h1:+60H3Cm91RnL9dpVGWDPHt0zTQImO9Vfqt9a4rSambI= -github.com/sagernet/sing-box v1.8.13 h1:M8mLo7AWDKV9d+Bq41fOz7B5CvS4+xhBR1tmwzrr7PY= -github.com/sagernet/sing-box v1.8.13/go.mod h1:Cb0FBO7r5UnQJQ7Ql0jgdJ8SmzCyFvCQ7eeXq2clTOI= -github.com/sagernet/sing-dns v0.1.14 h1:kxE/Ik3jMXmD3sXsdt9MgrNzLFWt64mghV+MQqzyf40= -github.com/sagernet/sing-dns v0.1.14/go.mod h1:AA+vZMNovuPN5i/sPnfF6756Nq94nzb5nXodMWbta5w= +github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= +github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= +github.com/quic-go/qtls-go1-20 v0.4.1 h1:D33340mCNDAIKBqXuAvexTNMUByrYmFYVfKfDN5nfFs= +github.com/quic-go/qtls-go1-20 v0.4.1/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k= +github.com/sagernet/quic-go v0.45.1-beta.2 h1:zkEeCbhdFFkrxKcuIRBtXNKci/1t2J/39QSG/sPvlmc= +github.com/sagernet/quic-go v0.45.1-beta.2/go.mod h1:+N3FqM9DAzOWfe64uxXuBejVJwX7DeW7BslzLO6N/xI= +github.com/sagernet/sing v0.5.0-alpha.12.0.20240717075530-332e47007567 h1:Sus0pCYn5KPOQmArXdRTtrml6XccSzV87FkFTDqATtU= +github.com/sagernet/sing v0.5.0-alpha.12.0.20240717075530-332e47007567/go.mod h1:ARkL0gM13/Iv5VCZmci/NuoOlePoIsW0m7BWfln/Hak= +github.com/sagernet/sing-box v1.9.4-0.20240718050851-e2b52d9664a6 h1:BlgwOJWF/Aq+AyTHA6RUbptK77xyP+n4uKmsgpDNCCA= +github.com/sagernet/sing-box v1.9.4-0.20240718050851-e2b52d9664a6/go.mod h1:pCo+ThuniLGPJvb/B/Q6uAmiahA4WnETHVqUJuNHpTs= +github.com/sagernet/sing-dns v0.3.0-beta.10 h1:Js61EjQXVpcu2VDegWEQTH1isCcVwJju8WEHYgG4tQ0= +github.com/sagernet/sing-dns v0.3.0-beta.10/go.mod h1:nXE6EYMXahB5DV3AcXYbFfuorqF7tbQ86kxweSxRKM4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= @@ -32,22 +44,26 @@ github.com/v2fly/v2ray-core/v5 v5.16.1 h1:hIuRzCJhmRYqCA76hGiNLkAHopgbNt91L871wl github.com/v2fly/v2ray-core/v5 v5.16.1/go.mod h1:3pWIBTmNagMKpzd9/QicXq/7JZCQt716GsGZdBNmYkU= go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M= go4.org/netipx v0.0.0-20231129151722-fdeea329fbba/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y= -golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= -golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= -golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY= +golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= +golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= +golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/tools v0.20.0 h1:hz/CVckiOxybQvFw6h7b/q80NTr9IUQb4s1IIzW7KNY= -golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= +golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.34.0 h1:Qo/qEd2RZPCf2nKuorzksSknv0d3ERwp1vFG38gSmH4= -google.golang.org/protobuf v1.34.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/main.go b/main.go index 1da7b1d521..5ecc1d6d8a 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,7 @@ package main import ( + "bufio" "context" "crypto/sha256" "encoding/hex" @@ -282,7 +283,7 @@ func mergeTags(data map[string][]geosite.Item) { println("merged cn categories: " + strings.Join(cnCodeList, ",")) } -func generate(release *github.RepositoryRelease, output string, cnOutput string, ruleSetOutput string) error { +func generate(release *github.RepositoryRelease, output string, cnOutput string, ruleSetOutput string, ruleSetUnstableOutput string) error { vData, err := download(release) if err != nil { return err @@ -300,7 +301,12 @@ func generate(release *github.RepositoryRelease, output string, cnOutput string, return err } defer outputFile.Close() - err = geosite.Write(outputFile, domainMap) + writer := bufio.NewWriter(outputFile) + err = geosite.Write(writer, domainMap) + if err != nil { + return err + } + err = writer.Flush() if err != nil { return err } @@ -316,12 +322,19 @@ func generate(release *github.RepositoryRelease, output string, cnOutput string, return err } defer cnOutputFile.Close() - err = geosite.Write(cnOutputFile, cnDomainMap) + writer.Reset(cnOutputFile) + err = geosite.Write(writer, cnDomainMap) + if err != nil { + return err + } + err = writer.Flush() if err != nil { return err } os.RemoveAll(ruleSetOutput) + os.RemoveAll(ruleSetUnstableOutput) err = os.MkdirAll(ruleSetOutput, 0o755) + err = os.MkdirAll(ruleSetUnstableOutput, 0o755) if err != nil { return err } @@ -340,17 +353,30 @@ func generate(release *github.RepositoryRelease, output string, cnOutput string, }, } srsPath, _ := filepath.Abs(filepath.Join(ruleSetOutput, "geosite-"+code+".srs")) - //os.Stderr.WriteString("write " + srsPath + "\n") - outputRuleSet, err := os.Create(srsPath) + unstableSRSPath, _ := filepath.Abs(filepath.Join(ruleSetUnstableOutput, "geosite-"+code+".srs")) + // os.Stderr.WriteString("write " + srsPath + "\n") + var ( + outputRuleSet *os.File + outputRuleSetUnstable *os.File + ) + outputRuleSet, err = os.Create(srsPath) + if err != nil { + return err + } + err = srs.Write(outputRuleSet, plainRuleSet, false) + outputRuleSet.Close() if err != nil { return err } - err = srs.Write(outputRuleSet, plainRuleSet) + outputRuleSetUnstable, err = os.Create(unstableSRSPath) + if err != nil { + return err + } + err = srs.Write(outputRuleSetUnstable, plainRuleSet, true) + outputRuleSetUnstable.Close() if err != nil { - outputRuleSet.Close() return err } - outputRuleSet.Close() } return nil } @@ -359,7 +385,7 @@ func setActionOutput(name string, content string) { os.Stdout.WriteString("::set-output name=" + name + "::" + content + "\n") } -func release(source string, destination string, output string, cnOutput string, ruleSetOutput string) error { +func release(source string, destination string, output string, cnOutput string, ruleSetOutput string, ruleSetOutputUnstable string) error { sourceRelease, err := fetch(source) if err != nil { return err @@ -374,7 +400,7 @@ func release(source string, destination string, output string, cnOutput string, return nil } } - err = generate(sourceRelease, output, cnOutput, ruleSetOutput) + err = generate(sourceRelease, output, cnOutput, ruleSetOutput, ruleSetOutputUnstable) if err != nil { return err } @@ -389,6 +415,7 @@ func main() { "geosite.db", "geosite-cn.db", "rule-set", + "rule-set-unstable", ) if err != nil { log.Fatal(err)