From 9dbf99fd07fc0cace0817116f9bbdcc599854b2a Mon Sep 17 00:00:00 2001 From: Phodal Huang Date: Sat, 6 Jan 2024 20:57:13 +0800 Subject: [PATCH] docs: reorg docs datastructure --- README.md | 6 +- _config.yml | 1 - docs/README.md | 13 - docs/_config.yml | 95 +++++ docs/adr/0001-support-for-sql-.md | 19 - docs/adr/0002-setup-powerful-build-system.md | 24 -- docs/adr/0003-add-publish-multiple-jar.md | 21 -- docs/adr/README.md | 5 - docs/home.md | 371 +++++++++++++++++++ gradle/publications.gradle | 2 +- 10 files changed, 470 insertions(+), 87 deletions(-) delete mode 100644 _config.yml delete mode 100644 docs/README.md create mode 100644 docs/_config.yml delete mode 100644 docs/adr/0001-support-for-sql-.md delete mode 100644 docs/adr/0002-setup-powerful-build-system.md delete mode 100644 docs/adr/0003-add-publish-multiple-jar.md delete mode 100644 docs/adr/README.md create mode 100644 docs/home.md diff --git a/README.md b/README.md index 471cd6be..4ab96cc2 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ > CHAPI (Common Hierarchical Abstract Parser and Information Converter) streamlines code analysis by converting diverse > language source code into a unified abstract model, simplifying cross-language development. Chapi -> 是一个通用层次抽象解析器与信息转换器，它可以将不同编程语言的源代码转换为统一的层次抽象模型。一个通用语言元信息转换器，能将不同语言转换为相同的模型。 +> 是一个通用层次抽象解析器与信息转换器，它可以将不同编程语言的源代码转换为统一的层次抽象模型。 Chapi => Cha Pi => Tea Pi => Tea π => 茶 π. See on in refs: [Tea if by sea, cha if by land](https://qz.com/1176962/map-how-the-word-tea-spread-over-land-and-sea-to-conquer-the-world/). @@ -103,8 +103,8 @@ Algol Family [https://wiki.c2.com/?AlgolFamily](https://wiki.c2.com/?AlgolFamily ``` dependencies { - implementation 'com.phodal.chapi:chapi-ast-java:0.2.0' - implementation 'com.phodal.chapi:chapi-domain:0.2.0' + implementation 'com.phodal.chapi:chapi-ast-java:2.2.2' + implementation 'com.phodal.chapi:chapi-domain:2.2.2' } ``` diff --git a/_config.yml b/_config.yml deleted file mode 100644 index 141db0f5..00000000 --- a/_config.yml +++ /dev/null @@ -1 +0,0 @@ -remote_theme: phodal/mifa-jekyll diff --git a/docs/README.md b/docs/README.md deleted file mode 100644 index 31442e8a..00000000 --- a/docs/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# Doc - - -## Deploy - -``` -gpg --list-signatures --keyid-format 0xshort -``` - -Sonatype Maven Issue: https://issues.sonatype.org/browse/OSSRH-78417 - -Signin: https://docs.gradle.org/current/userguide/signing_plugin.html - diff --git a/docs/_config.yml b/docs/_config.yml new file mode 100644 index 00000000..b17d80eb --- /dev/null +++ b/docs/_config.yml @@ -0,0 +1,95 @@ +remote_theme: pmarsceill/just-the-docs + +title: Chapi +description: CHAPI (Common Hierarchical Abstract Parser and Information Converter) streamlines code analysis by converting diverse language source code into a unified abstract model, simplifying cross-language development. + +heading_anchors: true + +footer_content: "This code is distributed under the MPL 2.0 license. See `LICENSE` in this directory." + +# Footer last edited timestamp +last_edit_timestamp: true # show or hide edit time - page must have `last_modified_date` defined in the frontmatter +last_edit_time_format: "%b %e %Y at %I:%M %p" # uses ruby's time format: https://ruby-doc.org/stdlib-2.7.0/libdoc/time/rdoc/Time.html + +# Footer "Edit this page on GitHub" link text +gh_edit_link: true # show or hide edit this page link +gh_edit_link_text: "Edit this page on GitHub." +gh_edit_repository: "https://github.com/phodal/chapi" # the github URL for your repo +gh_edit_branch: "master" # the branch that your docs are served from +gh_edit_source: docs # the source that your files originate from +gh_edit_view_mode: "tree" # "tree" or "edit" if you want the user to jump into the editor immediately + +callouts_level: quiet # or loud +callouts: + highlight: + color: yellow + important: + title: Important + color: blue + new: + title: New + color: green + note: + title: Note + color: purple + warning: + title: Warning + color: red + +# Enable or disable the site search +# Supports true (default) or false +search_enabled: true + +search: + # Split pages into sections that can be searched individually + # Supports 1 - 6, default: 2 + heading_level: 4 + # Maximum amount of previews per search result + # Default: 3 + previews: 3 + # Maximum amount of words to display before a matched word in the preview + # Default: 5 + preview_words_before: 5 + # Maximum amount of words to display after a matched word in the preview + # Default: 10 + preview_words_after: 10 + # Set the search token separator + # Default: /[\s\-/]+/ + # Example: enable support for hyphenated search words + tokenizer_separator: /[\s/]+/ + # Display the relative url in search results + # Supports true (default) or false + rel_url: true + # Enable or disable the search button that appears in the bottom right corner of every page + # Supports true or false (default) + button: false + + +# Back to top link +back_to_top: true +back_to_top_text: "Back to top" + +# Google Analytics Tracking (optional) +# e.g, UA-1234567-89 +# ga_tracking: G-JR0EZ3NLE1 +# ga_tracking_anonymize_ip: true # Use GDPR compliant Google Analytics settings (true by default) + +url: "https://chapi.phodal.com" # the base hostname & protocol for your site + +# Aux links for the upper right navigation +aux_links: + "View in on GitHub": + - "//github.com/phodal/chapi" + +# Makes Aux links open in a new tab. Default is false +aux_links_new_tab: true + +mermaid: + # Version of mermaid library + # Pick an available version from https://cdn.jsdelivr.net/npm/mermaid/ + version: "10.4.0" + + +plugins: + - jekyll-seo-tag + - jekyll-sitemap diff --git a/docs/adr/0001-support-for-sql-.md b/docs/adr/0001-support-for-sql-.md deleted file mode 100644 index 50e8ccf1..00000000 --- a/docs/adr/0001-support-for-sql-.md +++ /dev/null @@ -1,19 +0,0 @@ -# 1. support for sql? - -日期: 2020-02-11 - -## 状态 - -2020-02-11 提议 - -## 背景 - -在这里补充上下文... - -## 决策 - -在这里补充上决策信息... - -## 后果 - -在这里记录结果... diff --git a/docs/adr/0002-setup-powerful-build-system.md b/docs/adr/0002-setup-powerful-build-system.md deleted file mode 100644 index 3ac46bb1..00000000 --- a/docs/adr/0002-setup-powerful-build-system.md +++ /dev/null @@ -1,24 +0,0 @@ -# 2. setup powerful build system - -日期: 2020-02-11 - -## 状态 - -2020-02-11 提议 - -2020-02-20 完成 - -## 背景 - -当前的构建项目比较多，构建脚本比较简单，不能完成诸如下文中的功能： - - - 测试覆盖率 - - 包发布 - -## 决策 - -在这里补充上决策信息... - -## 后果 - -在这里记录结果... diff --git a/docs/adr/0003-add-publish-multiple-jar.md b/docs/adr/0003-add-publish-multiple-jar.md deleted file mode 100644 index 1f3c01f2..00000000 --- a/docs/adr/0003-add-publish-multiple-jar.md +++ /dev/null @@ -1,21 +0,0 @@ -# 3. add publish multiple jar - -日期: 2020-02-13 - -## 状态 - -2020-02-13 提议 - -2020-02-20 完成 - -## 背景 - -在这里补充上下文... - -## 决策 - -在这里补充上决策信息... - -## 后果 - -在这里记录结果... diff --git a/docs/adr/README.md b/docs/adr/README.md deleted file mode 100644 index 2defd017..00000000 --- a/docs/adr/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# 架构决策记录 - -* [1. support-for-sql-](0001-support-for-sql-.md) -* [2. setup-powerful-build-system](0002-setup-powerful-build-system.md) -* [3. add-publish-multiple-jar](0003-add-publish-multiple-jar.md) \ No newline at end of file diff --git a/docs/home.md b/docs/home.md new file mode 100644 index 00000000..e537e51e --- /dev/null +++ b/docs/home.md @@ -0,0 +1,371 @@ +--- +layout: default +title: Chapi +description: Chapi 是一个通用层次抽象解析器与信息转换器，它可以将不同编程语言的源代码转换为统一的层次抽象模型。 +nav_order: 1 +permalink: / +--- + +Chapi Logo + +[![Chapi CI](https://github.com/modernizing/chapi/actions/workflows/ci.yml/badge.svg)](https://github.com/modernizing/chapi/actions/workflows/ci.yml) +[![codecov](https://codecov.io/gh/phodal/chapi/graph/badge.svg?token=sdB5L3xbRv)](https://codecov.io/gh/phodal/chapi) +[![Maintainability](https://api.codeclimate.com/v1/badges/2af5f5168a9ceb2ebe9b/maintainability)](https://codeclimate.com/github/phodal/chapi/maintainability) +![Maven Central](https://img.shields.io/maven-central/v/com.phodal.chapi/chapi-domain) + +> CHAPI (Common Hierarchical Abstract Parser and Information Converter) streamlines code analysis by converting diverse +> language source code into a unified abstract model, simplifying cross-language development. Chapi +> 是一个通用层次抽象解析器与信息转换器，它可以将不同编程语言的源代码转换为统一的层次抽象模型。 + +Chapi => Cha Pi => Tea Pi => Tea π => 茶 π. See on in +refs: [Tea if by sea, cha if by land](https://qz.com/1176962/map-how-the-word-tea-spread-over-land-and-sea-to-conquer-the-world/). + +Chapi (pronoounce /tʃɑpi/) also pronounce XP in Chinese if you always call X in 叉. + +language stages: + +| Features | Java | Python | Go | Kotlin | TS/JS | C | C# | Scala | C++ | Rust | +|---------------|------|--------|----|--------|-------|----|----|-------|-----|------| +| http api decl | ✅ | 🆕 | 🆕 | ✅ | ✅ | 🆕 | 🆕 | | | 🆕 | +| syntax parse | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | 🆕 | ✅ | 🆕 | ✅ | +| function call | ✅ | 🆕 | 🆕 | ✅ | ✅ | | | | | ✅ | +| arch/package | ✅ | | | ✅ | ✅ | | | ✅ | | 🆕 | +| real world | ✅ | | | 🆕 | ✅ | | | | | | + +- [ArchGuard](https://github.com/archguard/archguard) - ArchGuard is an architecture workbench, also for architecture + governance, which can analysis architecture in container, component, code level, create architecture fitness functions, + and analysis system dependencies. +- [UnitGen](https://github.com/unit-mesh/unit-gen) is a code fine-tuning data framework that generates data from your + existing codebase. +- [ChocolateFactory](https://github.com/unit-mesh/chocolate-factory) is a cutting-edge LLM toolkit designed to empower + you in creating your very own AI assistant. + +PS: welcome to PR to send your projects + +## Language Information + +language versions（tested）: + +- Java: 8, 11, 17 +- TypeScript/JavaScript +- Kotlin +- Rust: v1.60.0 +- Python: 2, 3 + +// tier 1 languages +":chapi-ast-java", +":chapi-ast-typescript", + +// tier 2 languages +":chapi-ast-kotlin", +":chapi-ast-go", +":chapi-ast-python", +":chapi-ast-scala", + +// tier 3 languages +":chapi-ast-rust", +":chapi-ast-csharp", +":chapi-ast-c", +":chapi-ast-cpp", + +Language Family [wiki](https://en.wikipedia.org/wiki/First-class_function) + +Algol Family [https://wiki.c2.com/?AlgolFamily](https://wiki.c2.com/?AlgolFamily) + +| | Languages | plan support | +|------------|-----------------------------------------------|-------------------------| +| C family | C#, Java, Go, C, C++, Objective-C, Rust, ... | C++, C, Java, C#, Rust? | +| Functional | Scheme, Lisp, Clojure, Scala, ... | Scala | +| Scripting | Lua, PHP, JavaScript, Python, Perl, Ruby, ... | Python, JavaScript | +| Other | Fortran, Swift, Matlab, ... | Swift?, Fortran? | + +## Specify Rule + +**scan by twice**. In order to success get: + +- find data struct in same package + +### TypeScript + +1. PackageName will use resolvePath, package of `src/grammar/blbla.ts` is `@.grammar` +2. Function in file will use `default` as DataStructure.Name +3. export default `Object` in file will use `default` as FunctionName, and will belong to `default` DataStructure + +### C# issues + +- interpolated_string parse + issue：[official grammar](https://github.com/dotnet/roslyn/blob/main/src/Compilers/CSharp/Portable/Generated/CSharp.Generated.g4) + ，Antlr issues: [https://github.com/antlr/grammars-v4/issues/1146](https://github.com/antlr/grammars-v4/issues/1146) +- import analysis support + - in C#, import `namespace` can use call in namespace + +### Kotlin + +- need to implementation `warpTargetFullType` to got Class in same package. + +## Usage + +1. add to ``dependencies`` + +``` +dependencies { + implementation 'com.phodal.chapi:chapi-ast-java:2.2.2' + implementation 'com.phodal.chapi:chapi-domain:2.2.2' +} +``` + +### Usage + +```kotlin +import chapi.domain.core.CodeDataStruct +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.runBlocking +import org.archguard.scanner.core.sourcecode.LanguageSourceCodeAnalyser +import org.archguard.scanner.core.sourcecode.SourceCodeContext +import java.io.File + +class CSharpAnalyser(override val context: SourceCodeContext) + +private val client = context.client +private val impl = chapi.ast.csharpast.CSharpAnalyser() + +fun analyse(): List = runBlocking { + getFilesByPath(context.path) { + it.absolutePath.endsWith(".cs") + } + .map { async { analysisByFile(it) } }.awaitAll() + .flatten() + .also { client.saveDataStructure(it) } +} + +fun analysisByFile(file: File): List { + val codeContainer = impl.analysis(file.readContent(), file.name) + return codeContainer.Containers.flatMap { container -> + container.DataStructures.map { + it.apply { + it.Imports = codeContainer.Imports + it.FilePath = file.absolutePath + } + } + } +} +} + +``` + +## Examples + +examples Java source code: + +```java +package adapters.outbound.persistence.blog; + +public class BlogPO implements PersistenceObject { + @Override + public Blog toDomainModel() { + + } +} +``` + +examples output + +```json +{ + "Imports": [], + "Implements": [ + "PersistenceObject" + ], + "NodeName": "BlogPO", + "Extend": "", + "Type": "CLASS", + "FilePath": "", + "InOutProperties": [], + "Functions": [ + { + "IsConstructor": false, + "InnerFunctions": [], + "Position": { + "StartLine": 6, + "StartLinePosition": 133, + "StopLine": 8, + "StopLinePosition": 145 + }, + "Package": "", + "Name": "toDomainModel", + "MultipleReturns": [], + "Annotations": [ + { + "Name": "Override", + "KeyValues": [] + } + ], + "Extension": {}, + "Override": false, + "extensionMap": {}, + "Parameters": [], + "InnerStructures": [], + "ReturnType": "Blog", + "Modifiers": [], + "FunctionCalls": [] + } + ], + "Annotations": [], + "Extension": {}, + "Parameters": [], + "Fields": [], + "MultipleExtend": [], + "InnerStructures": [], + "Package": "adapters.outbound.persistence.blog", + "FunctionCalls": [] +} +``` + +## Development + +Syntax Parse Identify Rules: + +1. package name +2. import name +3. class / data struct + 1. struct name + 2. struct parameters + 3. function name + 4. return types + 5. function parameters +4. function + 1. function name + 2. return types + 3. function parameters +5. method call + 1. new instance call + 2. parameter call + 3. field call + +### Build Antlr Grammar + +1. setup Antlr: `brew install antlr` +2. run compile: `./scripts/compile-antlr.sh` + +### Data Structures + +``` +// for multiple project analysis +code_project +code_module + +// for package dependency analysis +code_package_info +code_dependency + +// package or file as dependency analysis +code_package +code_container + +// class-first or function-first +code_data_struct +code_function + +// function or class detail +code_annotation +code_field +code_import +code_member +code_position +code_property + +// method call information +code_call +``` + +## Development（Chinese Version） + +Env：Intellij IDEA、JDK 11+ + +1. Clone code：``git clone https://github.com/phodal/chapi`` + +2. Build：``./gradlew build`` + +### 参与开发 + +为了保证不易出现 bug，项目采用 TDD 的方式进行，即先编写对应的语法测试，然后实现代码。通过尽可能高的测试覆盖率，降低 bug 的出现。 + +项目主要由 domain + 各种语言的 AST + application 构建： + +- domain，构建统一的代码模型 +- 各语言 AST +- application，对外暴露的简易 API + +每个 AST 项目的入口是 ``xxAnalyser``，返回的是一个 CodeContainer，即代码容器。在非 C# 语言里，等同于 CodeFile，即代码文件。 + +CodeContainer 内对应的领域模型如下所示： + +``` +// class-first or function-first +code_data_struct // 类、struct、interface 等 +code_function // 函数。如果是头等函数的语言（first-class function”），会用 NodeName = "default" 包在 code_data_struct 模型中 + +// function or class detail +code_annotation // 注解 +code_field // 全局变量 +code_import // 包依赖 +code_member // 保留字段 +code_position // 位置信息 +code_property // 参数相关 + +// method call information +code_call // 函数调用，如 fmt.Println +``` + +### 加入开发 + +1. 寻找感兴趣的语言 / 添加新的语言 AST + +通过 TDD +的方式一点点实现下面的功能（可以考虑按顺序），参照示例见 [JavaFullIdentListenerTest.kt](https://github.com/phodal/chapi/blob/master/chapi-ast-java/src/test/kotlin/chapi/ast/javaast/JavaFullIdentListenerTest.kt)： + +1. package name +2. import name +3. class / data struct + 1. struct name + 2. struct parameters + 3. function name + 4. return types + 5. function parameters +4. function + 1. function name + 2. return types + 3. function parameters +5. method call + 1. new instance call + 2. parameter call + 3. field call + 4. other calls... + +### 提交信息格式 + +用于发布时，使用标准的 [CHANGELOG.md](CHANGELOG.md) + +`[Language]: `，示例：`feat(java): init python & go grammars Phodal Huang 2020/2/2, 5:01 PM` + +所有的 type 见： + +- build: 影响构建系统或外部依赖关系的更改（示例范围：gulp，broccoli，npm） +- ci: 更改我们的持续集成文件和脚本（示例范围：Travis，Circle，BrowserStack，SauceLabs） +- docs: 仅文档更改 +- feat: 一个新功能 +- fix: 修复错误 +- perf: 改进性能的代码更改 +- refactor: 代码更改，既不修复错误也不添加功能 +- style: 不影响代码含义的变化（空白，格式化，缺少分号等） +- test: 添加缺失测试或更正现有测试 + +License +--- + +[![Phodal's Idea](http://brand.phodal.com/shields/idea-small.svg)](http://ideas.phodal.com/) + +@2020 A [Phodal Huang](https://www.phodal.com)'s [Idea](http://github.com/phodal/ideas). This code is distributed under +the MPL license. See `LICENSE` in this directory. + diff --git a/gradle/publications.gradle b/gradle/publications.gradle index c19e4136..bb0627d1 100644 --- a/gradle/publications.gradle +++ b/gradle/publications.gradle @@ -67,7 +67,7 @@ bintray { issueTrackerUrl = 'https://github.com/phodal/chapi/issues' labels = ["meta-language", "meta-data", "meta-programming", "meta-analysis", "meta-model", "modeling"] githubRepo = "phodal/chapi" - githubReleaseNotesFile = 'README.md' + githubReleaseNotesFile = 'home.md' licenses = ['MPL-2.0'] vcsUrl = "https://github.com/phodal/chapi" description = "Chapi (Common Hierarchical Abstract Parser and Information Converter) streamlines code analysis by converting diverse language source code into a unified abstract model, simplifying cross-language development."