diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 596745ce7..3b9d3927a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ env: jobs: build: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - name: Checkout uses: actions/checkout@v2 diff --git a/README.md b/README.md index 7fd53987f..1f9d36769 100644 --- a/README.md +++ b/README.md @@ -7,12 +7,17 @@ This script can be automatically generate something what you needed every day. [![Flag Counter](https://s04.flagcounter.com/countxl/jtlp/bg_FFFFFF/txt_000000/border_CCCCCC/columns_2/maxflags_10/viewers_3/labels_1/pageviews_1/flags_0/percent_0/)](https://info.flagcounter.com/jtlp) +[![Powered by DartNode](https://github.com/bin456789/reinstall/assets/7548515/435d6740-bcdd-4f3a-a196-2f60ae397f17)](https://dartnode.com "Powered by DartNode - Free VPS for Open Source") +[![Powered by DartNode](https://dartnode.com/branding/DN-Open-Source-sm.png)](https://dartnode.com "Powered by DartNode - Free VPS for Open Source") + ## 🍭 Lists - [`adlist/adlist.txt`](https://raw.githubusercontent.com/pexcn/daily/gh-pages/adlist/adlist.txt) - [`adblock/adblock.conf`](https://raw.githubusercontent.com/pexcn/daily/gh-pages/adblock/adblock.conf) - [`chnroute/chnroute.txt`](https://raw.githubusercontent.com/pexcn/daily/gh-pages/chnroute/chnroute.txt) - [`chnroute/chnroute6.txt`](https://raw.githubusercontent.com/pexcn/daily/gh-pages/chnroute/chnroute6.txt) +- [`hkgroute/hkgroute.txt`](https://raw.githubusercontent.com/pexcn/daily/gh-pages/hkgroute/hkgroute.txt) +- [`hkgroute/hkgroute6.txt`](https://raw.githubusercontent.com/pexcn/daily/gh-pages/hkgroute/hkgroute6.txt) - [`chinalist/chinalist.txt`](https://raw.githubusercontent.com/pexcn/daily/gh-pages/chinalist/chinalist.txt) - [`gfwlist/gfwlist.txt`](https://raw.githubusercontent.com/pexcn/daily/gh-pages/gfwlist/gfwlist.txt) - [`pac/gfwlist.pac`](https://raw.githubusercontent.com/pexcn/daily/gh-pages/pac/gfwlist.pac) diff --git a/generate.sh b/generate.sh index 8b6449a26..450a6af2c 100755 --- a/generate.sh +++ b/generate.sh @@ -23,6 +23,10 @@ post() { } main() { + # toplist + mkdir -p dist/toplist/ + scripts/toplist.sh> dist/toplist/toplist.txt + # ad domain list scripts/adlist/adlist.sh @@ -40,6 +44,9 @@ main() { # chnroute scripts/chnroute/chnroute.sh + # hkgroute + scripts/hkgroute/hkgroute.sh + # pac scripts/pac/pac.sh diff --git a/scripts/_pre_/01-toplist.sh b/scripts/_pre_/01-toplist.sh deleted file mode 100755 index 71c28b46d..000000000 --- a/scripts/_pre_/01-toplist.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -e -set -o pipefail - -CUR_DIR=$(pwd) -TMP_DIR=$(mktemp -d /tmp/toplist.XXXXXX) - -SRC_URL="https://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip" -DEST_FILE="dist/toplist/toplist.txt" - -gen_list() { - cd $TMP_DIR - - curl -sSL $SRC_URL | - # unzip - gunzip | - # extract domain - awk -F ',' '{print 2ドル}' | - # convert to unix format - sed $'s/\r$//'> toplist.txt - - cd $CUR_DIR -} - -copy_dest() { - install -D -m 644 $TMP_DIR/toplist.txt $DEST_FILE -} - -clean_up() { - rm -r $TMP_DIR -} - -gen_list -copy_dest -clean_up diff --git a/scripts/_pre_/50-tools.sh b/scripts/_pre_/50-tools.sh index e4edd2787..1c6efb952 100755 --- a/scripts/_pre_/50-tools.sh +++ b/scripts/_pre_/50-tools.sh @@ -7,5 +7,6 @@ cd ../.. # https://github.com/afpd/netaggregate/pull/1 cd tools/netaggregate +sed -i '/#include /a #include ' netaggregate.hpp make --silent cd ../.. diff --git a/scripts/chinalist/chinalist.sh b/scripts/chinalist/chinalist.sh index ef9e87d1a..a2195759a 100755 --- a/scripts/chinalist/chinalist.sh +++ b/scripts/chinalist/chinalist.sh @@ -5,7 +5,6 @@ CUR_DIR=$(pwd) TMP_DIR=$(mktemp -d /tmp/chinalist.XXXXXX) SRC_URL_1="https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf" -SRC_URL_2="https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/google.china.conf" SRC_URL_3="https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf" SRC_URL_4="https://raw.githubusercontent.com/pexcn/daily-extras/master/chinalist-extras.txt" SRC_FILE="$CUR_DIR/dist/toplist/toplist.txt" @@ -15,7 +14,6 @@ fetch_src() { cd $TMP_DIR curl -sSL $SRC_URL_1 -o apple.conf - curl -sSL $SRC_URL_2 -o google.conf curl -sSL $SRC_URL_3 -o china.conf curl -sSL $SRC_URL_4 -o chinalist-extras.txt cp $SRC_FILE . @@ -26,7 +24,7 @@ fetch_src() { gen_list() { cd $TMP_DIR - cat apple.conf google.conf china.conf | + cat apple.conf china.conf | # remove empty lines sed '/^[[:space:]]*$/d' | # remove comment lines diff --git a/scripts/chnroute/chnroute.sh b/scripts/chnroute/chnroute.sh index 7ca4fbbba..13b1c7e94 100755 --- a/scripts/chnroute/chnroute.sh +++ b/scripts/chnroute/chnroute.sh @@ -4,7 +4,7 @@ set -o pipefail CUR_DIR=$(pwd) TMP_DIR=$(mktemp -d /tmp/chnroute.XXXXXX) -SRC_URL_1="https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-latest" +SRC_URL_1="https://ftp.apnic.net/stats/apnic/delegated-apnic-latest" SRC_URL_2="https://raw.githubusercontent.com/17mon/china_ip_list/master/china_ip_list.txt" DEST_FILE_1="dist/chnroute/chnroute.txt" DEST_FILE_2="dist/chnroute/chnroute6.txt" diff --git a/scripts/hkgroute/hkgroute.sh b/scripts/hkgroute/hkgroute.sh new file mode 100755 index 000000000..493ca68b7 --- /dev/null +++ b/scripts/hkgroute/hkgroute.sh @@ -0,0 +1,57 @@ +#!/bin/bash -e +set -o pipefail + +CUR_DIR=$(pwd) +TMP_DIR=$(mktemp -d /tmp/hkgroute.XXXXXX) + +SRC_URL_1="https://ftp.apnic.net/stats/apnic/delegated-apnic-latest" +DEST_FILE_1="dist/hkgroute/hkgroute.txt" +DEST_FILE_2="dist/hkgroute/hkgroute6.txt" + +fetch_src() { + cd $TMP_DIR + + curl -sSL $SRC_URL_1 -o apnic.txt + + cd $CUR_DIR +} + +gen_list_v4() { + cd $TMP_DIR + + # convert to cidr format + cat apnic.txt | grep ipv4 | grep HK | awk -F\| '{ printf("%s/%d\n", 4,ドル 32-log(5ドル)/log(2)) }'> apnic-v4.tmp + + # ipv4 cidr merge + cat apnic-v4.tmp | $CUR_DIR/tools/ip-dedup/obj/ip-dedup -4> hkgroute.txt + + cd $CUR_DIR +} + +gen_list_v6() { + cd $TMP_DIR + + # convert to cidr format + cat apnic.txt | grep ipv6 | grep HK | awk -F\| '{ printf("%s/%d\n", 4,ドル 5ドル) }'> apnic-v6.tmp + + # ipv6 cidr merge + cat apnic-v6.tmp | $CUR_DIR/tools/ip-dedup/obj/ip-dedup -6> hkgroute6.txt + + cd $CUR_DIR +} + +copy_dest() { + install -D -m 644 $TMP_DIR/hkgroute.txt $DEST_FILE_1 + install -D -m 644 $TMP_DIR/hkgroute6.txt $DEST_FILE_2 +} + +clean_up() { + rm -r $TMP_DIR + echo "[$(basename 0ドル .sh)]: done." +} + +fetch_src +gen_list_v4 +gen_list_v6 +copy_dest +clean_up diff --git a/scripts/toplist.sh b/scripts/toplist.sh new file mode 100755 index 000000000..5f23464db --- /dev/null +++ b/scripts/toplist.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +TOPLIST_URL="https://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip" + +curl -L $TOPLIST_URL | + gunzip | + # extract domain + cut -d',' -f2 | + # convert to unix format + sed 's/\r$//' diff --git a/template/pac/gfwlist.template b/template/pac/gfwlist.template index 52a7e473e..ea14f15fb 100644 --- a/template/pac/gfwlist.template +++ b/template/pac/gfwlist.template @@ -1,50 +1,91 @@ -const proxy = "SOCKS5 127.0.0.1:1080;"; -const direct = "DIRECT;"; +const proxy = "SOCKS5 127.0.0.1:1080"; +const direct = "DIRECT"; -const hasOwnProperty = Object.hasOwnProperty; +const hasOwn = Object.hasOwn || function (obj, prop) { + return Object.prototype.hasOwnProperty.call(obj, prop); +}; -const gfwlist_domains = { +const gfwlistDomains = { ___GFWLIST_PLACEHOLDER___ }; -function is_china_domain(domain) { - return !!dnsDomainIs(domain, ".cn"); +const CACHE_LIMIT = 4096; +const cache = Object.create(null); +const cacheRing = new Array(CACHE_LIMIT); +let cachePos = 0; +let cacheSize = 0; + +function cacheGet(host) { + return cache[host]; } -function match_domains(domain, domains) { - let suffix; - let pos = domain.lastIndexOf('.'); - pos = domain.lastIndexOf('.', pos - 1); - while (1) { - if (pos <= 0) { - return hasOwnProperty.call(domains, domain); +function cachePut(host, val) { + if (cache[host] !== undefined) { + cache[host] = val; + return; + } + if (cacheSize < CACHE_LIMIT) { + cacheRing[cacheSize++] = host; + } else { + const old = cacheRing[cachePos]; + if (old !== undefined) { + delete cache[old]; } - suffix = domain.substring(pos + 1); - if (hasOwnProperty.call(domains, suffix)) { + cacheRing[cachePos] = host; + cachePos = (cachePos + 1) % CACHE_LIMIT; + } + cache[host] = val; +} + +function isCn(host) { + const len = host.length; + let end = len; + if (len> 0 && host.charCodeAt(len - 1) === 46) { // '.' + end = len - 1; + } + return end> 3 + && host.charCodeAt(end - 3) === 46 // '.' + && host.charCodeAt(end - 2) === 99 // 'c' + && host.charCodeAt(end - 1) === 110; // 'n' +} + +function matchDomains(host, domains) { + if (hasOwn(domains, host)) return true; + + let last = host.lastIndexOf('.'); + if (last <= 0) return false; + + let pos = host.lastIndexOf('.', last - 1); + while (pos> 0) { + if (hasOwn(domains, host.substring(pos + 1))) { return true; } - pos = domain.lastIndexOf('.', pos - 1); + pos = host.lastIndexOf('.', pos - 1); } + + return false; } -/** - * @return {string} Connect via direct or proxy. - */ function FindProxyForURL(url, host) { if (typeof host === 'undefined' || isPlainHostName(host) === true || host === '127.0.0.1' - || host === 'localhost') { + || host === 'localhost' + || host === '::1') { return direct; } - if (is_china_domain(host) === true) { - return direct; + const cached = cacheGet(host); + if (cached !== undefined) { + return cached; } - if (match_domains(host, gfwlist_domains) === true) { - return proxy; + if (isCn(host) === true) { + cachePut(host, direct); + return direct; } - return direct; + const result = matchDomains(host, gfwlistDomains) === true ? proxy : direct; + cachePut(host, result); + return result; } diff --git a/template/pac/whitelist.template b/template/pac/whitelist.template index c3afb0e23..0a0fd6ba9 100644 --- a/template/pac/whitelist.template +++ b/template/pac/whitelist.template @@ -1,104 +1,203 @@ -const proxy = "SOCKS5 127.0.0.1:1080;"; -const direct = "DIRECT;"; +const proxy = "SOCKS5 127.0.0.1:1080"; +const direct = "DIRECT"; -const hasOwnProperty = Object.hasOwnProperty; +const hasOwn = Object.hasOwn || function (obj, prop) { + return Object.prototype.hasOwnProperty.call(obj, prop); +}; -const china_domains = { +const chinaDomains = { ___CHINALIST_PLACEHOLDER___ }; -const subnet_ips = [ - 0, 16777216, // 0.0.0.0 -> 0.255.255.255 - 167772160, 184549376, // 10.0.0.0/8 -> 10.255.255.255 - 1681915904, 1686110208, // 100.64.0.0/10 -> 100.127.255.255 - 2130706432, 2147483648, // 127.0.0.0/8 -> 127.255.255.255 - -1442971648, -1442906112, // 169.254.0.0/16 -> 169.254.255.255 - -1408237568, -1407188992, // 172.16.0.0/12 -> 172.31.255.255 - -1073741824, -1073741568, // 192.0.0.0/24 -> 192.0.0.255 - -1073741312, -1073741056, // 192.0.2.0/24 -> 192.0.2.255 - -1071660032, -1071659776, // 192.31.196.0/24 -> 192.31.196.255 - -1070284544, -1070284288, // 192.52.193.0/24 -> 192.52.193.255 - -1067949312, -1067949056, // 192.88.99.0/24 -> 192.88.99.255 - -1062731776, -1062666240, // 192.168.0.0/16 -> 192.168.255.255 - -1062260736, -1062260480, // 192.175.48.0/24 -> 192.175.48.255 - -971898880, -971767808, // 198.18.0.0/15 -> 198.19.255.255 - -969710592, -969710336, // 198.51.100.0/24 -> 198.51.100.255 - -889163520, -889163264, // 203.0.113.0/24 -> 203.0.113.255 - -536870912, 0 // 224.0.0.0/3 -> 255.255.255.255 +const SUBNET_CIDRS = [ + "0.0.0.0/8", + "10.0.0.0/8", + "100.64.0.0/10", + "127.0.0.0/8", + "169.254.0.0/16", + "172.16.0.0/12", + "192.0.0.0/24", + "192.0.2.0/24", + "192.31.196.0/24", + "192.52.193.0/24", + "192.88.99.0/24", + "192.168.0.0/16", + "192.175.48.0/24", + "198.18.0.0/15", + "198.51.100.0/24", + "203.0.113.0/24", + "224.0.0.0/4", + "240.0.0.0/4", + "255.255.255.255/32", ]; -function is_ipv4(host) { - const regex = /^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(:([0-9]|[1-9]\d|[1-9]\d{2}|[1-9]\d{3}|[1-5]\d{4}|6[0-4]\d{3}|65[0-4]\d{2}|655[0-2]\d|6553[0-5]))?$/g; - if (regex.test(host)) { - return true; +const CACHE_LIMIT = 4096; +const cache = Object.create(null); +const cacheRing = new Array(CACHE_LIMIT); +let cachePos = 0; +let cacheSize = 0; + +function cacheGet(host) { + return cache[host]; +} + +function cachePut(host, val) { + if (cache[host] !== undefined) { + cache[host] = val; + return; } + if (cacheSize < CACHE_LIMIT) { + cacheRing[cacheSize++] = host; + } else { + const old = cacheRing[cachePos]; + if (old !== undefined) { + delete cache[old]; + } + cacheRing[cachePos] = host; + cachePos = (cachePos + 1) % CACHE_LIMIT; + } + cache[host] = val; } -function convert_address(ipchars) { - if (ipchars.indexOf(':') !== -1) { - ipchars = ipchars.split(':')[0]; +function isCn(host) { + const len = host.length; + let end = len; + if (len> 0 && host.charCodeAt(len - 1) === 46) { // '.' + end = len - 1; } - const bytes = ipchars.split('.'); - const result = ((bytes[0] & 0xff) << 24) | - ((bytes[1] & 0xff) << 16) | - ((bytes[2] & 0xff) << 8) | - (bytes[3] & 0xff); - return result; + return end> 3 + && host.charCodeAt(end - 3) === 46 // '.' + && host.charCodeAt(end - 2) === 99 // 'c' + && host.charCodeAt(end - 1) === 110; // 'n' } -function is_china_domain(domain) { - return !!dnsDomainIs(domain, ".cn"); +function ipv4ToUint(host) { + const colon = host.indexOf(':'); + if (colon !== -1) { + if (host.indexOf('.') === -1) return null; // host is ipv6 + host = host.substring(0, colon); + } + + let acc = 0; + let octet = 0; + let digits = 0; + let parts = 0; + let leadingZero = false; + + for (let i = 0; i < host.length; i++) { + const c = host.charCodeAt(i); + + if (c>= 48 && c <= 57) { + if (digits === 0) { + leadingZero = (c === 48); + } + digits++; + octet = octet * 10 + (c - 48); + if (octet> 255) return null; + continue; + } + + if (c !== 46) return null; // only '.' allowed + if (digits === 0) return null; // empty octet + if (leadingZero && digits> 1) return null; // reject leading zeros like "01" + + acc = acc * 256 + octet; + parts++; + if (parts> 3) return null; + + octet = 0; + digits = 0; + leadingZero = false; + } + + if (digits === 0) return null; + if (leadingZero && digits> 1) return null; + + acc = acc * 256 + octet; + if (parts !== 3) return null; + + return acc; } -function match_domains(domain, domains) { - let suffix; - let pos = domain.lastIndexOf('.'); - pos = domain.lastIndexOf('.', pos - 1); - while (1) { - if (pos <= 0) { - return hasOwnProperty.call(domains, domain); +const ipTable = new Array(256); +(function initIpTable() { + for (let i = 0; i < 256; i++) { + ipTable[i] = []; + } + for (let i = 0; i < SUBNET_CIDRS.length; i++) { + const cidr = SUBNET_CIDRS[i]; + const slash = cidr.indexOf('/'); + if (slash < 0) continue; + const ipStr = cidr.substring(0, slash); + const prefix = parseInt(cidr.substring(slash + 1), 10); + if (!(prefix>= 0 && prefix <= 32)) continue; + const base = ipv4ToUint(ipStr); + if (base === null) continue; + const mask = prefix === 0 ? 0 : ((0xFFFFFFFF << (32 - prefix))>>> 0); + const start = (base & mask)>>> 0; + const end = start + Math.pow(2, 32 - prefix); // [start, end) + const pStart = (start>>> 24) & 0xFF; + const pEnd = ((end - 1)>>> 24) & 0xFF; + for (let p = pStart; p <= pEnd; p++) { + ipTable[p].push(start, end); } - suffix = domain.substring(pos + 1); - if (hasOwnProperty.call(domains, suffix)) { + } +})(); + +function matchSubnet(ip) { + const ranges = ipTable[(ip>>> 24) & 0xFF]; + for (let i = 0; i < ranges.length; i += 2) { + if (ranges[i] <= ip && ip < ranges[i + 1]) { return true; } - pos = domain.lastIndexOf('.', pos - 1); } + return false; } -function match_subnet_ips(ip, ips) { - for (let i = 0; i < 28; i += 2) { - if (ips[i] <= ip && ip < ips[i + 1]) { +function matchDomains(host, domains) { + if (hasOwn(domains, host)) return true; + + let last = host.lastIndexOf('.'); + if (last <= 0) return false; + + let pos = host.lastIndexOf('.', last - 1); + while (pos> 0) { + if (hasOwn(domains, host.substring(pos + 1))) { return true; } + pos = host.lastIndexOf('.', pos - 1); } + + return false; } -/** - * @return {string} Connect via direct or proxy. - */ function FindProxyForURL(url, host) { if (typeof host === 'undefined' || isPlainHostName(host) === true || host === '127.0.0.1' - || host === 'localhost') { + || host === 'localhost' + || host === '::1') { return direct; } - if (is_china_domain(host) === true) { - return direct; + const cached = cacheGet(host); + if (cached !== undefined) { + return cached; } - if (match_domains(host, china_domains) === true) { - return direct; - } - - if (is_ipv4(host) === true) { - const ip = convert_address(host); - if (match_subnet_ips(ip, subnet_ips) === true) { - return direct; + let result = proxy; + if (isCn(host) === true || matchDomains(host, chinaDomains) === true) { + result = direct; + } else { + const c0 = host.length> 0 ? host.charCodeAt(0) : 0; + if (c0>= 48 && c0 <= 57) { + const ip = ipv4ToUint(host); + if (ip !== null && matchSubnet(ip) === true) { + result = direct; + } } } - return proxy; + cachePut(host, result); + return result; }

AltStyle γ«γ‚ˆγ£γ¦ε€‰ζ›γ•γ‚ŒγŸγƒšγƒΌγ‚Έ (->γ‚ͺγƒͺγ‚ΈγƒŠγƒ«) /