-
-
Notifications
You must be signed in to change notification settings - Fork 2.4k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
9dac025
commit 8285874
Showing
6 changed files
with
191 additions
and
142 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,2 @@ | ||
| requests-html==0.10.0 | ||
| retry==0.9.2 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,105 @@ | ||
| #!/usr/bin/env python | ||
| # -*- coding:utf-8 -*- | ||
| # | ||
| # Author : XueWeiHan | ||
| # E-mail : [email protected] | ||
| # Date : 2025-01-16 15:27 | ||
| # Desc : 公共函数 | ||
| import os | ||
| import json | ||
| from typing import Any, Optional | ||
| from datetime import datetime, timezone, timedelta | ||
|
|
||
| from retry import retry | ||
|
|
||
| GITHUB_URLS = [ | ||
| 'alive.github.com', 'api.github.com', 'assets-cdn.github.com', | ||
| 'avatars.githubusercontent.com', 'avatars0.githubusercontent.com', | ||
| 'avatars1.githubusercontent.com', 'avatars2.githubusercontent.com', | ||
| 'avatars3.githubusercontent.com', 'avatars4.githubusercontent.com', | ||
| 'avatars5.githubusercontent.com', 'camo.githubusercontent.com', | ||
| 'central.github.com', 'cloud.githubusercontent.com', 'codeload.github.com', | ||
| 'collector.github.com', 'desktop.githubusercontent.com', | ||
| 'favicons.githubusercontent.com', 'gist.github.com', | ||
| 'github-cloud.s3.amazonaws.com', 'github-com.s3.amazonaws.com', | ||
| 'github-production-release-asset-2e65be.s3.amazonaws.com', | ||
| 'github-production-repository-file-5c1aeb.s3.amazonaws.com', | ||
| 'github-production-user-asset-6210df.s3.amazonaws.com', 'github.blog', | ||
| 'github.com', 'github.community', 'github.githubassets.com', | ||
| 'github.global.ssl.fastly.net', 'github.io', 'github.map.fastly.net', | ||
| 'githubstatus.com', 'live.github.com', 'media.githubusercontent.com', | ||
| 'objects.githubusercontent.com', 'pipelines.actions.githubusercontent.com', | ||
| 'raw.githubusercontent.com', 'user-images.githubusercontent.com', | ||
| 'vscode.dev', 'education.github.com', 'private-user-images.githubusercontent.com' | ||
| ] | ||
|
|
||
| HOSTS_TEMPLATE = """# GitHub520 Host Start | ||
| {content} | ||
| # Update time: {update_time} | ||
| # Update url: https://raw.hellogithub.com/hosts | ||
| # Star me: https://github.com/521xueweihan/GitHub520 | ||
| # GitHub520 Host End\n""" | ||
|
|
||
|
|
||
| @retry(tries=3) | ||
| def get_json(session: Any) -> Optional[list]: | ||
| url = 'https://raw.hellogithub.com/hosts.json' | ||
| try: | ||
| rs = session.get(url) | ||
| data = json.loads(rs.text) | ||
| return data | ||
| except Exception as ex: | ||
| print(f"get: {url}, error: {ex}") | ||
| raise Exception | ||
|
|
||
|
|
||
| def write_file(hosts_content: str, update_time: str) -> bool: | ||
| output_doc_file_path = os.path.join(os.path.dirname(__file__), "README.md") | ||
| template_path = os.path.join(os.path.dirname(__file__), | ||
| "README_template.md") | ||
| write_host_file(hosts_content) | ||
| if os.path.exists(output_doc_file_path): | ||
| with open(output_doc_file_path, "r") as old_readme_fb: | ||
| old_content = old_readme_fb.read() | ||
| if old_content: | ||
| old_hosts = old_content.split("```bash")[1].split("```")[0].strip() | ||
| old_hosts = old_hosts.split("# Update time:")[0].strip() | ||
| hosts_content_hosts = hosts_content.split("# Update time:")[ | ||
| 0].strip() | ||
| if old_hosts == hosts_content_hosts: | ||
| print("host not change") | ||
| return False | ||
|
|
||
| with open(template_path, "r") as temp_fb: | ||
| template_str = temp_fb.read() | ||
| hosts_content = template_str.format(hosts_str=hosts_content, | ||
| update_time=update_time) | ||
| with open(output_doc_file_path, "w") as output_fb: | ||
| output_fb.write(hosts_content) | ||
| return True | ||
|
|
||
|
|
||
| def write_host_file(hosts_content: str) -> None: | ||
| output_file_path = os.path.join(os.path.dirname(__file__), 'hosts') | ||
| with open(output_file_path, "w") as output_fb: | ||
| output_fb.write(hosts_content) | ||
|
|
||
|
|
||
| def write_json_file(hosts_list: list) -> None: | ||
| output_file_path = os.path.join(os.path.dirname(__file__), 'hosts.json') | ||
| with open(output_file_path, "w") as output_fb: | ||
| json.dump(hosts_list, output_fb) | ||
|
|
||
|
|
||
| def write_hosts_content(content: str, content_list: list) -> str: | ||
| if not content: | ||
| return "" | ||
| update_time = datetime.now(timezone.utc).astimezone( | ||
| timezone(timedelta(hours=8))).replace(microsecond=0).isoformat() | ||
| hosts_content = HOSTS_TEMPLATE.format(content=content, | ||
| update_time=update_time) | ||
| has_change = write_file(hosts_content, update_time) | ||
| if has_change: | ||
| write_json_file(content_list) | ||
| return hosts_content |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -5,95 +5,26 @@ | |
| # E-mail : [email protected] | ||
| # Date : 2020-05-19 15:27 | ||
| # Desc : 获取最新的 GitHub 相关域名对应 IP | ||
| import os | ||
| import re | ||
| import json | ||
| import argparse | ||
| from typing import Any, Optional | ||
|
|
||
| from datetime import datetime, timezone, timedelta | ||
| from datetime import datetime | ||
|
|
||
| from pythonping import ping | ||
| from requests_html import HTMLSession | ||
| from retry import retry | ||
|
|
||
| GITHUB_URLS = [ | ||
| 'alive.github.com', 'api.github.com', 'assets-cdn.github.com', | ||
| 'avatars.githubusercontent.com', 'avatars0.githubusercontent.com', | ||
| 'avatars1.githubusercontent.com', 'avatars2.githubusercontent.com', | ||
| 'avatars3.githubusercontent.com', 'avatars4.githubusercontent.com', | ||
| 'avatars5.githubusercontent.com', 'camo.githubusercontent.com', | ||
| 'central.github.com', 'cloud.githubusercontent.com', 'codeload.github.com', | ||
| 'collector.github.com', 'desktop.githubusercontent.com', | ||
| 'favicons.githubusercontent.com', 'gist.github.com', | ||
| 'github-cloud.s3.amazonaws.com', 'github-com.s3.amazonaws.com', | ||
| 'github-production-release-asset-2e65be.s3.amazonaws.com', | ||
| 'github-production-repository-file-5c1aeb.s3.amazonaws.com', | ||
| 'github-production-user-asset-6210df.s3.amazonaws.com', 'github.blog', | ||
| 'github.com', 'github.community', 'github.githubassets.com', | ||
| 'github.global.ssl.fastly.net', 'github.io', 'github.map.fastly.net', | ||
| 'githubstatus.com', 'live.github.com', 'media.githubusercontent.com', | ||
| 'objects.githubusercontent.com', 'pipelines.actions.githubusercontent.com', | ||
| 'raw.githubusercontent.com', 'user-images.githubusercontent.com', | ||
| 'vscode.dev', 'education.github.com', 'private-user-images.githubusercontent.com' | ||
| ] | ||
|
|
||
| HOSTS_TEMPLATE = """# GitHub520 Host Start | ||
| {content} | ||
| # Update time: {update_time} | ||
| # Update url: https://raw.hellogithub.com/hosts | ||
| # Star me: https://github.com/521xueweihan/GitHub520 | ||
| # GitHub520 Host End\n""" | ||
|
|
||
|
|
||
| def write_file(hosts_content: str, update_time: str) -> bool: | ||
| output_doc_file_path = os.path.join(os.path.dirname(__file__), "README.md") | ||
| template_path = os.path.join(os.path.dirname(__file__), | ||
| "README_template.md") | ||
| write_host_file(hosts_content) | ||
| if os.path.exists(output_doc_file_path): | ||
| with open(output_doc_file_path, "r") as old_readme_fb: | ||
| old_content = old_readme_fb.read() | ||
| if old_content: | ||
| old_hosts = old_content.split("```bash")[1].split("```")[0].strip() | ||
| old_hosts = old_hosts.split("# Update time:")[0].strip() | ||
| hosts_content_hosts = hosts_content.split("# Update time:")[ | ||
| 0].strip() | ||
| if old_hosts == hosts_content_hosts: | ||
| print("host not change") | ||
| return False | ||
|
|
||
| with open(template_path, "r") as temp_fb: | ||
| template_str = temp_fb.read() | ||
| hosts_content = template_str.format(hosts_str=hosts_content, | ||
| update_time=update_time) | ||
| with open(output_doc_file_path, "w") as output_fb: | ||
| output_fb.write(hosts_content) | ||
| return True | ||
|
|
||
|
|
||
| def write_host_file(hosts_content: str) -> None: | ||
| output_file_path = os.path.join(os.path.dirname(__file__), 'hosts') | ||
| with open(output_file_path, "w") as output_fb: | ||
| output_fb.write(hosts_content) | ||
|
|
||
|
|
||
| def write_json_file(hosts_list: list) -> None: | ||
| output_file_path = os.path.join(os.path.dirname(__file__), 'hosts.json') | ||
| with open(output_file_path, "w") as output_fb: | ||
| json.dump(hosts_list, output_fb) | ||
| from common import GITHUB_URLS, write_hosts_content | ||
|
|
||
|
|
||
| def get_best_ip(ip_list: list) -> str: | ||
| ping_timeout = 2 | ||
| ping_timeout = 1 | ||
| best_ip = '' | ||
| min_ms = ping_timeout * 500 | ||
| min_ms = ping_timeout * 1000 | ||
| ip_set = set(ip_list) | ||
| for ip in ip_set: | ||
| ping_result = ping(ip, timeout=ping_timeout) | ||
| print(ping_result.rtt_avg_ms) | ||
| if ping_result.rtt_avg_ms == ping_timeout * 500: | ||
| print(f'ping {ip} {ping_result.rtt_avg_ms} ms') | ||
| if ping_result.rtt_avg_ms == ping_timeout * 1000: | ||
| # 超时认为 IP 失效 | ||
| continue | ||
| else: | ||
|
|
@@ -103,18 +34,6 @@ def get_best_ip(ip_list: list) -> str: | |
| return best_ip | ||
|
|
||
|
|
||
| @retry(tries=3) | ||
| def get_json(session: Any) -> Optional[list]: | ||
| url = 'https://raw.hellogithub.com/hosts.json' | ||
| try: | ||
| rs = session.get(url) | ||
| data = json.loads(rs.text) | ||
| return data | ||
| except Exception as ex: | ||
| print(f"get: {url}, error: {ex}") | ||
| raise Exception | ||
|
|
||
|
|
||
| @retry(tries=3) | ||
| def get_ip(session: Any, github_url: str) -> Optional[str]: | ||
| url = f'https://sites.ipaddress.com/{github_url}' | ||
|
|
@@ -130,52 +49,33 @@ def get_ip(session: Any, github_url: str) -> Optional[str]: | |
| if best_ip: | ||
| return best_ip | ||
| else: | ||
| raise Exception(f"url: {github_url}, ipaddress empty") | ||
| raise Exception("url: {github_url}, ipaddress empty") | ||
| except Exception as ex: | ||
| print(f"get: {url}, error: {ex}") | ||
| print("get: {url}, error: {ex}") | ||
| raise Exception | ||
|
|
||
|
|
||
| def main(verbose=False, model='server') -> None: | ||
| if verbose: | ||
| print('Start script.') | ||
| def main() -> None: | ||
| current_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S') | ||
| print(f'{current_time} - Start script.') | ||
| session = HTMLSession() | ||
| content = "" | ||
| if model == 'server': | ||
| content_list = [] | ||
| for index, github_url in enumerate(GITHUB_URLS): | ||
| try: | ||
| ip = get_ip(session, github_url) | ||
|
|
||
| content += ip.ljust(30) + github_url + "\n" | ||
| content_list.append((ip, github_url,)) | ||
| except Exception: | ||
| continue | ||
| if verbose: | ||
| print(f'process url: {index + 1}/{len(GITHUB_URLS)}') | ||
| else: | ||
| content_list = get_json(session) | ||
| for item in content_list: | ||
| content += item[0].ljust(30) + item[1] + "\n" | ||
| content_list = [] | ||
| for index, github_url in enumerate(GITHUB_URLS): | ||
| try: | ||
| ip = get_ip(session, github_url) | ||
|
|
||
| content += ip.ljust(30) + github_url + "\n" | ||
| content_list.append((ip, github_url,)) | ||
| except Exception: | ||
| continue | ||
| print(f'Process url: {index + 1}/{len(GITHUB_URLS)}, {github_url}') | ||
|
|
||
| if not content: | ||
| return | ||
| update_time = datetime.utcnow().astimezone( | ||
| timezone(timedelta(hours=8))).replace(microsecond=0).isoformat() | ||
| hosts_content = HOSTS_TEMPLATE.format(content=content, | ||
| update_time=update_time) | ||
| has_change = write_file(hosts_content, update_time) | ||
| if has_change: | ||
| write_json_file(content_list) | ||
| if verbose: | ||
| print(hosts_content) | ||
| print('End script.') | ||
| write_hosts_content(content, content_list) | ||
| # print(hosts_content) | ||
| current_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S') | ||
| print(f'{current_time} - End script.') | ||
|
|
||
|
|
||
| if __name__ == '__main__': | ||
| parser = argparse.ArgumentParser(description="Run fetch ips script") | ||
| # 2. 定义参数 | ||
| parser.add_argument('model', metavar='model', type=str, nargs='?') | ||
| # 3. 解析命令行 | ||
| args = parser.parse_args() | ||
| main(True, args.model) | ||
| main() | ||
Oops, something went wrong.