From d549e175e03e9982eb68c6c9460bcf32bfdfd699 Mon Sep 17 00:00:00 2001 From: Abel Soares Siqueira Date: Tue, 11 Jun 2024 11:57:00 +0200 Subject: [PATCH] Apply COPIERTemplate 0.5.3 with minimum optional questions --- .JuliaFormatter.toml | 9 +- .copier-answers.yml | 11 + .github/dependabot.yml | 7 + .github/workflows/CompatHelper.yml | 6 +- .github/workflows/Docs.yml | 6 +- .github/workflows/ReusableTest.yml | 52 +++ .github/workflows/Test.yml | 62 +--- docs/make.jl | 40 +- docs/src/90-contributing.md | 25 ++ docs/src/90-developer.md | 133 +++++++ docs/src/{reference.md => 90-reference.md} | 6 +- lychee.toml | 11 + src/exceptions.jl | 55 ++- src/fmtsql.jl | 153 ++++---- src/influxdb.jl | 58 +-- src/parsers.jl | 253 +++++++------ src/pipeline.jl | 328 ++++++++--------- test/runtests.jl | 4 +- test/test-parsers.jl | 112 +++--- test/test-pipeline.jl | 401 ++++++++++----------- 20 files changed, 971 insertions(+), 761 deletions(-) create mode 100644 .copier-answers.yml create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/ReusableTest.yml create mode 100644 docs/src/90-contributing.md create mode 100644 docs/src/90-developer.md rename docs/src/{reference.md => 90-reference.md} (52%) create mode 100644 lychee.toml diff --git a/.JuliaFormatter.toml b/.JuliaFormatter.toml index 960c4bb..15a73f2 100644 --- a/.JuliaFormatter.toml +++ b/.JuliaFormatter.toml @@ -2,12 +2,17 @@ align_assignment = true align_matrix = true align_pair_arrow = true align_struct_field = true +always_for_in = true +annotate_untyped_fields_with_any = false conditional_to_if = true +for_in_replacement = "in" format_docstrings = false format_markdown = false -indent = 4 +import_to_using = true +indent = 2 margin = 100 normalize_line_endings = "unix" remove_extra_newlines = true separate_kwargs_with_semicolon = true -verbose = true +whitespace_ops_in_indices = true +whitespace_typedefs = true diff --git a/.copier-answers.yml b/.copier-answers.yml new file mode 100644 index 0000000..7ed0956 --- /dev/null +++ b/.copier-answers.yml @@ -0,0 +1,11 @@ +# Changes here will be overwritten by Copier +AnswerStrategy: minimum +AuthorEmail: fatkasuvayu+linux@gmail.com +AuthorName: Suvayu Ali +JuliaMinVersion: '1.6' +License: Apache-2.0 +PackageName: TulipaIO +PackageOwner: TulipaEnergy +PackageUUID: 7b3808b7-0819-42d4-885c-978ba173db11 +_commit: v0.5.3 +_src_path: https://github.com/abelsiqueira/COPIERTemplate.jl diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..700707c --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,7 @@ +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" # Location of package manifests + schedule: + interval: "weekly" diff --git a/.github/workflows/CompatHelper.yml b/.github/workflows/CompatHelper.yml index 7825cfe..210e56f 100644 --- a/.github/workflows/CompatHelper.yml +++ b/.github/workflows/CompatHelper.yml @@ -3,7 +3,7 @@ name: CompatHelper on: schedule: - - cron: 0 0 * * * + - cron: 0 0 * * * # Every day at 00:00 UTC workflow_dispatch: permissions: @@ -19,11 +19,13 @@ jobs: run: which julia continue-on-error: true - name: Install Julia, but only if it is not already available in the PATH - uses: julia-actions/setup-julia@v1 + uses: julia-actions/setup-julia@v2 with: version: "1" arch: ${{ runner.arch }} if: steps.julia_in_path.outcome != 'success' + - name: Use Julia cache + uses: julia-actions/cache@v2 - name: "Add the General registry via Git" run: | import Pkg diff --git a/.github/workflows/Docs.yml b/.github/workflows/Docs.yml index b7ec126..583ee42 100644 --- a/.github/workflows/Docs.yml +++ b/.github/workflows/Docs.yml @@ -29,10 +29,12 @@ jobs: name: Documentation runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: julia-actions/setup-julia@v1 + - uses: actions/checkout@v4 + - uses: julia-actions/setup-julia@v2 with: version: "1" + - name: Use Julia cache + uses: julia-actions/cache@v2 - run: | julia --project=docs -e ' using Pkg diff --git a/.github/workflows/ReusableTest.yml b/.github/workflows/ReusableTest.yml new file mode 100644 index 0000000..12431f2 --- /dev/null +++ b/.github/workflows/ReusableTest.yml @@ -0,0 +1,52 @@ +name: Reusable test + +on: + workflow_call: + inputs: + version: + required: false + type: string + default: "1" + os: + required: false + type: string + default: ubuntu-latest + arch: + required: false + type: string + default: x64 + allow_failure: + required: false + type: boolean + default: false + run_codecov: + required: false + type: boolean + default: false + secrets: + codecov_token: + required: true + +jobs: + test: + name: Julia ${{ inputs.version }} - ${{ inputs.os }} - ${{ inputs.arch }} - ${{ github.event_name }} + runs-on: ${{ inputs.os }} + continue-on-error: ${{ inputs.allow_failure }} + + steps: + - uses: actions/checkout@v4 + - uses: julia-actions/setup-julia@v2 + with: + version: ${{ inputs.version }} + arch: ${{ inputs.arch }} + - name: Use Julia cache + uses: julia-actions/cache@v2 + - uses: julia-actions/julia-buildpkg@v1 + - uses: julia-actions/julia-runtest@v1 + - uses: julia-actions/julia-processcoverage@v1 + if: ${{ inputs.run_codecov }} + - uses: codecov/codecov-action@v4 + if: ${{ inputs.run_codecov }} + with: + file: lcov.info + token: ${{ secrets.codecov_token }} diff --git a/.github/workflows/Test.yml b/.github/workflows/Test.yml index f9ecaee..fd1b573 100644 --- a/.github/workflows/Test.yml +++ b/.github/workflows/Test.yml @@ -4,10 +4,6 @@ on: push: branches: - main - paths: - - "src/**" - - "test/**" - - "*.toml" tags: ["*"] pull_request: branches: @@ -17,63 +13,29 @@ on: - "test/**" - "*.toml" types: [opened, synchronize, reopened] - -concurrency: - # Skip intermediate builds: always. - # Cancel intermediate builds: only if it is a pull request build. - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} + workflow_dispatch: jobs: test: - name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} - runs-on: ${{ matrix.os }} + uses: ./.github/workflows/ReusableTest.yml + with: + os: ${{ matrix.os }} + version: ${{ matrix.version }} + arch: ${{ matrix.arch }} + allow_failure: ${{ matrix.allow_failure }} + run_codecov: ${{ matrix.version == '1' && matrix.os == 'ubuntu-latest' }} + secrets: + codecov_token: ${{ secrets.CODECOV_TOKEN }} strategy: fail-fast: false matrix: version: - "1.6" - "1" - - "nightly" os: - ubuntu-latest - - macOS-latest - - windows-latest + #- macOS-latest + #- windows-latest arch: - x64 allow_failure: [false] - include: - - version: "nightly" - os: ubuntu-latest - arch: x64 - allow_failure: true - - version: "nightly" - os: macOS-latest - arch: x64 - allow_failure: true - - version: "nightly" - os: windows-latest - arch: x64 - allow_failure: true - steps: - - uses: actions/checkout@v3 - - uses: julia-actions/setup-julia@v1 - with: - version: ${{ matrix.version }} - arch: ${{ matrix.arch }} - - uses: actions/cache@v1 - env: - cache-name: cache-artifacts - with: - path: ~/.julia/artifacts - key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} - restore-keys: | - ${{ runner.os }}-test-${{ env.cache-name }}- - ${{ runner.os }}-test- - ${{ runner.os }}- - - uses: julia-actions/julia-buildpkg@v1 - - uses: julia-actions/julia-runtest@v1 - - uses: julia-actions/julia-processcoverage@v1 - - uses: codecov/codecov-action@v1 - with: - file: lcov.info diff --git a/docs/make.jl b/docs/make.jl index d375866..5cc0b92 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -3,19 +3,35 @@ using Documenter DocMeta.setdocmeta!(TulipaIO, :DocTestSetup, :(using TulipaIO); recursive = true) +const page_rename = Dict("developer.md" => "Developer docs") # Without the numbers + +function nice_name(file) + file = replace(file, r"^[0-9]*-" => "") + if haskey(page_rename, file) + return page_rename[file] + end + return splitext(file)[1] |> x -> replace(x, "-" => " ") |> titlecase +end + makedocs(; - modules = [TulipaIO], - doctest = true, - linkcheck = true, - authors = "Suvayu Ali and contributors", - repo = "https://github.com/TulipaEnergy/TulipaIO.jl/blob/{commit}{path}#{line}", - sitename = "TulipaIO.jl", - format = Documenter.HTML(; - prettyurls = get(ENV, "CI", "false") == "true", - canonical = "https://TulipaEnergy.github.io/TulipaIO.jl", - assets = ["assets/style.css"], - ), - pages = ["Home" => "index.md", "Reference" => "reference.md"], + modules = [TulipaIO], + doctest = true, + linkcheck = false, # Rely on Lint.yml/lychee for the links + authors = "Suvayu Ali and contributors", + repo = "https://github.com/TulipaEnergy/TulipaIO.jl/blob/{commit}{path}#{line}", + sitename = "TulipaIO.jl", + format = Documenter.HTML(; + prettyurls = true, + canonical = "https://TulipaEnergy.github.io/TulipaIO.jl", + assets = ["assets/style.css"], + ), + pages = [ + "Home" => "index.md" + [ + nice_name(file) => file for + file in readdir(joinpath(@__DIR__, "src")) if file != "index.md" && splitext(file)[2] == ".md" + ] + ], ) deploydocs(; repo = "github.com/TulipaEnergy/TulipaIO.jl", push_preview = true) diff --git a/docs/src/90-contributing.md b/docs/src/90-contributing.md new file mode 100644 index 0000000..9c6f225 --- /dev/null +++ b/docs/src/90-contributing.md @@ -0,0 +1,25 @@ +# [Contributing guidelines](@id contributing) + +First of all, thanks for the interest! + +We welcome all kinds of contribution, including, but not limited to code, documentation, examples, configuration, issue creating, etc. + +Be polite and respectful, and follow the code of conduct. + +## Bug reports and discussions + +If you think you found a bug, feel free to open an [issue](https://github.com/TulipaEnergy/TulipaIO.jl/issues). +Focused suggestions and requests can also be opened as issues. +Before opening a pull request, start an issue or a discussion on the topic, please. + +## Working on an issue + +If you found an issue that interests you, comment on that issue what your plans are. +If the solution to the issue is clear, you can immediately create a pull request (see below). +Otherwise, say what your proposed solution is and wait for a discussion around it. + +!!! tip + Feel free to ping us after a few days if there are no responses. + +If your solution involves code (or something that requires running the package locally), check the [developer documentation](90-developer.md). +Otherwise, you can use the GitHub interface directly to create your pull request. diff --git a/docs/src/90-developer.md b/docs/src/90-developer.md new file mode 100644 index 0000000..573273b --- /dev/null +++ b/docs/src/90-developer.md @@ -0,0 +1,133 @@ +# [Developer documentation](@id dev_docs) + +!!! note "Contributing guidelines" + If you haven't, please read the [Contributing guidelines](90-contributing.md) first. + +If you want to make contributions to this package that involves code, then this guide is for you. + +## First time clone + +!!! tip "If you have writing rights" + If you have writing rights, you don't have to fork. Instead, simply clone and skip ahead. Whenever **upstream** is mentioned, use **origin** instead. + +If this is the first time you work with this repository, follow the instructions below to clone the repository. + +1. Fork this repo +2. Clone your repo (this will create a `git remote` called `origin`) +3. Add this repo as a remote: + + ```bash + git remote add upstream https://github.com/TulipaEnergy/TulipaIO.jl + ``` + +This will ensure that you have two remotes in your git: `origin` and `upstream`. +You will create branches and push to `origin`, and you will fetch and update your local `main` branch from `upstream`. + +## Linting and formatting + +Install a plugin on your editor to use [EditorConfig](https://editorconfig.org). +This will ensure that your editor is configured with important formatting settings. + +## Testing + +As with most Julia packages, you can just open Julia in the repository folder, activate the environment, and run `test`: + +```julia-repl +julia> # press ] +pkg> activate . +pkg> test +``` + +## Working on a new issue + +We try to keep a linear history in this repo, so it is important to keep your branches up-to-date. + +1. Fetch from the remote and fast-forward your local main + + ```bash + git fetch upstream + git switch main + git merge --ff-only upstream/main + ``` + +2. Branch from `main` to address the issue (see below for naming) + + ```bash + git switch -c 42-add-answer-universe + ``` + +3. Push the new local branch to your personal remote repository + + ```bash + git push -u origin 42-add-answer-universe + ``` + +4. Create a pull request to merge your remote branch into the org main. + +### Branch naming + +- If there is an associated issue, add the issue number. +- If there is no associated issue, **and the changes are small**, add a prefix such as "typo", "hotfix", "small-refactor", according to the type of update. +- If the changes are not small and there is no associated issue, then create the issue first, so we can properly discuss the changes. +- Use dash separated imperative wording related to the issue (e.g., `14-add-tests`, `15-fix-model`, `16-remove-obsolete-files`). + +### Commit message + +- Use imperative or present tense, for instance: *Add feature* or *Fix bug*. +- Have informative titles. +- When necessary, add a body with details. +- If there are breaking changes, add the information to the commit message. + +### Before creating a pull request + +!!! tip "Atomic git commits" + Try to create "atomic git commits" (recommended reading: [The Utopic Git History](https://blog.esciencecenter.nl/the-utopic-git-history-d44b81c09593)). + +- Make sure the tests pass. + +- Fetch any `main` updates from upstream and rebase your branch, if necessary: + + ```bash + git fetch upstream + git rebase upstream/main BRANCH_NAME + ``` + +- Then you can open a pull request and work with the reviewer to address any issues. + +## Building and viewing the documentation locally + +Following the latest suggestions, we recommend using `LiveServer` to build the documentation. +Here is how you do it: + +1. Run `julia --project=docs` to open Julia in the environment of the docs. +1. If this is the first time building the docs + 1. Press `]` to enter `pkg` mode + 1. Run `pkg> dev .` to use the development version of your package + 1. Press backspace to leave `pkg` mode +1. Run `julia> using LiveServer` +1. Run `julia> servedocs()` + +## Making a new release + +To create a new release, you can follow these simple steps: + +- Create a branch `release-x.y.z` +- Update `version` in `Project.toml` +- Update the `CHANGELOG.md`: + - Rename the section "Unreleased" to "[x.y.z] - yyyy-mm-dd" (i.e., version under brackets, dash, and date in ISO format) + - Add a new section on top of it named "Unreleased" + - Add a new link in the bottom for version "x.y.z" + - Change the "[unreleased]" link to use the latest version - end of line, `vx.y.z ... HEAD`. +- Create a commit "Release vx.y.z", push, create a PR, wait for it to pass, merge the PR. +- Go back to main screen and click on the latest commit (link: ) +- At the bottom, write `@JuliaRegistrator register` + +After that, you only need to wait and verify: + +- Wait for the bot to comment (should take < 1m) with a link to a RP to the registry +- Follow the link and wait for a comment on the auto-merge +- The comment should said all is well and auto-merge should occur shortly +- After the merge happens, TagBot will trigger and create a new GitHub tag. Check on +- After the release is create, a "docs" GitHub action will start for the tag. +- After it passes, a deploy action will run. +- After that runs, the [stable docs](https://TulipaEnergy.github.io/TulipaIO.jl/stable) should be updated. Check them and look for the version number. diff --git a/docs/src/reference.md b/docs/src/90-reference.md similarity index 52% rename from docs/src/reference.md rename to docs/src/90-reference.md index 4ed4f11..a7df265 100644 --- a/docs/src/reference.md +++ b/docs/src/90-reference.md @@ -1,15 +1,15 @@ -# Reference +# [Reference](@id reference) ## Contents ```@contents -Pages = ["reference.md"] +Pages = ["90-reference.md"] ``` ## Index ```@index -Pages = ["reference.md"] +Pages = ["90-reference.md"] ``` ```@autodocs diff --git a/lychee.toml b/lychee.toml new file mode 100644 index 0000000..399e29b --- /dev/null +++ b/lychee.toml @@ -0,0 +1,11 @@ +exclude = [ + "@ref", + "^https://github.com/.*/releases/tag/v.*$", + "^https://doi.org/FIXME$", + "^https://TulipaEnergy.github.io/TulipaIO.jl/stable$", + "zenodo.org/badge/DOI/FIXME$" +] + +exclude_path = [ + "docs/build" +] diff --git a/src/exceptions.jl b/src/exceptions.jl index a7a68ff..395aff6 100644 --- a/src/exceptions.jl +++ b/src/exceptions.jl @@ -1,42 +1,41 @@ import DuckDB: DB struct FileNotFoundError <: Exception - file::String - msg::String - function FileNotFoundError(file) - if ispath(file) - new(file, "$(file): exists, but not a regular file") - else - new(file, "$(file): file not found") - end + file::String + msg::String + function FileNotFoundError(file) + if ispath(file) + new(file, "$(file): exists, but not a regular file") + else + new(file, "$(file): file not found") end + end end struct DirectoryNotFoundError <: Exception - dir::String - msg::String - function DirectoryNotFoundError(dir) - if ispath(dir) - new(dir, "$(dir): exists, but not a directory") - else - new(dir, "$(dir): directory not found") - end + dir::String + msg::String + function DirectoryNotFoundError(dir) + if ispath(dir) + new(dir, "$(dir): exists, but not a directory") + else + new(dir, "$(dir): directory not found") end + end end struct TableNotFoundError <: Exception - con::DB - tbl::String - msg::String - TableNotFoundError(con, tbl) = new(con, tbl, "$(tbl): table not found in $(con)") + con::DB + tbl::String + msg::String + TableNotFoundError(con, tbl) = new(con, tbl, "$(tbl): table not found in $(con)") end struct NeitherTableNorFileError <: Exception - con::DB - src::String - msg::String - NeitherTableNorFileError(con, src) = - new(con, src, "$(src): neither table ($con) nor file found") + con::DB + src::String + msg::String + NeitherTableNorFileError(con, src) = new(con, src, "$(src): neither table ($con) nor file found") end Base.showerror(io::IO, exc::FileNotFoundError) = print(io, exc.msg) @@ -45,9 +44,9 @@ Base.showerror(io::IO, exc::TableNotFoundError) = print(io, exc.msg) Base.showerror(io::IO, exc::NeitherTableNorFileError) = print(io, exc.msg) struct InvalidWhereCondition <: Exception - expr::Any - msg::String - InvalidWhereCondition(expr) = new(expr, "Does not match format (lhs, op, rhs): $(expr)") + expr::Any + msg::String + InvalidWhereCondition(expr) = new(expr, "Does not match format (lhs, op, rhs): $(expr)") end Base.showerror(io::IO, exc::InvalidWhereCondition) = print(io, exc.msg) diff --git a/src/fmtsql.jl b/src/fmtsql.jl index 7835945..bd8b71a 100644 --- a/src/fmtsql.jl +++ b/src/fmtsql.jl @@ -8,110 +8,109 @@ sprintf(fmt::String, args...) = format(Format(fmt), args...) # quote literals appropriately for SQL fmt_quote(item) = "$(item)" -fmt_quote(item::Union{AbstractString,AbstractChar}) = "'$(item)'" +fmt_quote(item::Union{AbstractString, AbstractChar}) = "'$(item)'" fmt_quote(::Missing) = missing function fmt_opts(source::String; opts...) - _src = '?' in source ? "$source" : "'$(source)'" - join(["$(_src)"; [join(p, "=") for p in opts]], ", ") + _src = '?' in source ? "$source" : "'$(source)'" + join(["$(_src)"; [join(p, "=") for p in opts]], ", ") end function reader(source::String) - _, ext = splitext(source) - if ext in (".csv", ".parquet", ".json") - return "read_$(ext[2:end])_auto" - elseif '?' in source - # FIXME: how to support other file formats? - return "read_csv_auto" - else - error("$(ext[2:end]): unsupported input file '$(source)'") - end + _, ext = splitext(source) + if ext in (".csv", ".parquet", ".json") + return "read_$(ext[2:end])_auto" + elseif '?' in source + # FIXME: how to support other file formats? + return "read_csv_auto" + else + error("$(ext[2:end]): unsupported input file '$(source)'") + end end function fmt_read(source::String; opts...) - sprintf("%s(%s)", reader(source), fmt_opts(source; opts...)) + sprintf("%s(%s)", reader(source), fmt_opts(source; opts...)) end function fmt_select(source::String; cols...) - alts = if length(cols) > 0 - exclude = join(keys(cols), ", ") - include = join([sprintf("%s AS %s", fmt_quote(p[2]), p[1]) for p in cols], ", ") - "EXCLUDE ($exclude), $include" - else - "" - end - "SELECT * $alts FROM $source" + alts = if length(cols) > 0 + exclude = join(keys(cols), ", ") + include = join([sprintf("%s AS %s", fmt_quote(p[2]), p[1]) for p in cols], ", ") + "EXCLUDE ($exclude), $include" + else + "" + end + "SELECT * $alts FROM $source" end function fmt_join( - from_subquery::String, - join_subquery::String; - on::Vector{Symbol}, - cols::Vector{Symbol}, - fill::Bool, - fill_values::Union{Missing,Dict} = missing, + from_subquery::String, + join_subquery::String; + on::Vector{Symbol}, + cols::Vector{Symbol}, + fill::Bool, + fill_values::Union{Missing, Dict} = missing, ) - exclude = join(cols, ", ") - if fill - # e.g.: IFNULL(t2.investable, t1.investable) AS investable - if ismissing(fill_values) - include = join(map(c -> "IFNULL(t2.$c, t1.$c) AS $c", cols), ", ") - else - include = join( - map( - c -> begin - fill_value = - get(fill_values, c, missing) |> fmt_quote |> v -> coalesce(v, "t1.$c") - "IFNULL(t2.$c, $fill_value) AS $c" - end, - cols, - ), - ", ", - ) - end + exclude = join(cols, ", ") + if fill + # e.g.: IFNULL(t2.investable, t1.investable) AS investable + if ismissing(fill_values) + include = join(map(c -> "IFNULL(t2.$c, t1.$c) AS $c", cols), ", ") else - include = join(map(c -> "t2.$c", cols), ", ") + include = join( + map( + c -> begin + fill_value = get(fill_values, c, missing) |> fmt_quote |> v -> coalesce(v, "t1.$c") + "IFNULL(t2.$c, $fill_value) AS $c" + end, + cols, + ), + ", ", + ) end - select_ = "SELECT t1.* EXCLUDE ($exclude), $include" + else + include = join(map(c -> "t2.$c", cols), ", ") + end + select_ = "SELECT t1.* EXCLUDE ($exclude), $include" - join_on = join(map(c -> "t1.$c = t2.$c", on), " AND ") - from_ = "FROM $from_subquery t1 LEFT JOIN $join_subquery t2 ON ($join_on)" + join_on = join(map(c -> "t1.$c = t2.$c", on), " AND ") + from_ = "FROM $from_subquery t1 LEFT JOIN $join_subquery t2 ON ($join_on)" - "$(select_)\n$(from_)" + "$(select_)\n$(from_)" end _ops = (:(==), :(>), :(:<), :(>=), :(<=), :(!=), :%, :in) _ops_map = (; :(!=) => "<>", :% => "LIKE", :in => "IN") macro where_(exprs...) - xs = [] - for e in exprs - if !isa(e, Expr) || e.head != :call || length(e.args) != 3 - throw(InvalidWhereCondition(e)) - end - op, lhs, rhs = e.args - if !(op in _ops) - # FIXME: more specific exception - throw(InvalidWhereCondition(e)) - end - op = op in keys(_ops_map) ? _ops_map[op] : "$op" - if op == "IN" - rhs = eval(rhs) # FIXME: eval in invocation environment - if isa(rhs, AbstractRange) - op = "BETWEEN" - rhs = sprintf("%s AND %s", fmt_quote(rhs.start), fmt_quote(rhs.stop)) - elseif length(rhs) > 1 - rhs = sprintf("(%s)", join([sprintf(fmt_quote(rhs[1]), i) for i in rhs], ", ")) - else - # FIXME: clearer exception - throw(TypeError("$(rhs): not a range or array type")) - end - else - rhs = fmt_quote(rhs) - end - append!(xs, [sprintf("(%s %s %s)", lhs, op, rhs)]) + xs = [] + for e in exprs + if !isa(e, Expr) || e.head != :call || length(e.args) != 3 + throw(InvalidWhereCondition(e)) + end + op, lhs, rhs = e.args + if !(op in _ops) + # FIXME: more specific exception + throw(InvalidWhereCondition(e)) + end + op = op in keys(_ops_map) ? _ops_map[op] : "$op" + if op == "IN" + rhs = eval(rhs) # FIXME: eval in invocation environment + if isa(rhs, AbstractRange) + op = "BETWEEN" + rhs = sprintf("%s AND %s", fmt_quote(rhs.start), fmt_quote(rhs.stop)) + elseif length(rhs) > 1 + rhs = sprintf("(%s)", join([sprintf(fmt_quote(rhs[1]), i) for i in rhs], ", ")) + else + # FIXME: clearer exception + throw(TypeError("$(rhs): not a range or array type")) + end + else + rhs = fmt_quote(rhs) end - join(xs, " AND ") + append!(xs, [sprintf("(%s %s %s)", lhs, op, rhs)]) + end + join(xs, " AND ") end end # module FmtSQL diff --git a/src/influxdb.jl b/src/influxdb.jl index 25e172c..03432d6 100644 --- a/src/influxdb.jl +++ b/src/influxdb.jl @@ -1,7 +1,7 @@ module InfluxDB -import JSON3 -import HTTP +using JSON3: JSON3 +using HTTP: HTTP import DataFrames as DF import Dates: DateTime @@ -9,41 +9,41 @@ import Dates: DateTime # or keeping an open connection, it just remembers the connection # details struct InfluxDBClient - host::String - database::String - port::Int - path::String - username::String - password::String + host::String + database::String + port::Int + path::String + username::String + password::String end InfluxDBClient(host::String, database::String) = - InfluxDBClient(host, database, 8086, "query", "", "") + InfluxDBClient(host, database, 8086, "query", "", "") function query( - client::InfluxDBClient, - measurement::String, - time_range_start::DateTime, - time_range_end::DateTime, + client::InfluxDBClient, + measurement::String, + time_range_start::DateTime, + time_range_end::DateTime, ) - # NOTE: the query is not escaped, so no untrusted input should be accepted here - db_query = "SELECT time, value FROM \"$measurement\" WHERE time >= $time_range_start AND time <= $time_range_end" - url_params = ["db" => client.database, "q" => db_query] - uri = HTTP.URI(; - scheme = "http", - host = client.host, - path = client.path, - port = client.port, - query = url_params, - ) + # NOTE: the query is not escaped, so no untrusted input should be accepted here + db_query = "SELECT time, value FROM \"$measurement\" WHERE time >= $time_range_start AND time <= $time_range_end" + url_params = ["db" => client.database, "q" => db_query] + uri = HTTP.URI(; + scheme = "http", + host = client.host, + path = client.path, + port = client.port, + query = url_params, + ) - response = HTTP.get(uri) - parsed = JSON3.read(response.body) + response = HTTP.get(uri) + parsed = JSON3.read(response.body) - rows = parsed["results"][1]["series"][1]["values"] - columns = [[x[1] for x in rows], [x[2] for x in rows]] - df = DF.DataFrame(columns, parsed["results"][1]["series"][1]["columns"]) - return df + rows = parsed["results"][1]["series"][1]["values"] + columns = [[x[1] for x in rows], [x[2] for x in rows]] + df = DF.DataFrame(columns, parsed["results"][1]["series"][1]["columns"]) + return df end end diff --git a/src/parsers.jl b/src/parsers.jl index edb7287..d413619 100644 --- a/src/parsers.jl +++ b/src/parsers.jl @@ -1,6 +1,6 @@ import Base: merge -import JSON3 +using JSON3: JSON3 import PrettyTables: pretty_table export read_esdl_json @@ -20,14 +20,14 @@ Returns reduced value, or `sentinel` """ function reduce_unless(fn, itr; init, sentinel) - res = init - for i in itr - res = fn(res, i) - if res == sentinel - return sentinel - end + res = init + for i in itr + res = fn(res, i) + if res == sentinel + return sentinel end - return res + end + return res end """ @@ -44,21 +44,21 @@ Returns resolved value """ function resolve!(field, values, errs) - nonull = Iterators.filter(i -> i != nothing, values) |> collect - num = length(nonull) - if num == 0 - return nothing - elseif num > 1 - # check equality when more than one non-null values - iseq = reduce_unless( - (r, i) -> r ? isequal(i...) : false, # proceed iff equal - [nonull[i:i+1] for i = 1:num if i < num]; # paired iteration - init = true, - sentinel = false, - ) - !iseq && push!(errs, field) - end - return nonull[1] # unused on error, return for type-stability + nonull = Iterators.filter(i -> i != nothing, values) |> collect + num = length(nonull) + if num == 0 + return nothing + elseif num > 1 + # check equality when more than one non-null values + iseq = reduce_unless( + (r, i) -> r ? isequal(i...) : false, # proceed iff equal + [nonull[i:(i + 1)] for i in 1:num if i < num]; # paired iteration + init = true, + sentinel = false, + ) + !iseq && push!(errs, field) + end + return nonull[1] # unused on error, return for type-stability end """ @@ -70,24 +70,24 @@ error with a summary of the fields with conflicting values. """ function merge(args...; names = []) - nargs = length(args) - if nargs == 1 - return args[1] - end - - tp = typeof(args[1]) - errs = [] - res = tp((resolve!(f, map(t -> getfield(t, f), args), errs) for f in fieldnames(tp))...) - - if length(errs) > 0 - msg = "Following fields have conflicting values:\n" - # filter fields with errors for all arguments - data = map(t -> map(f -> getfield(t, f), errs), args) |> collect |> x -> hcat(errs, x...) - hdr = ["fields", ((length(names) == 0) ? (1:length(args)) : names)...] - msg *= pretty_table(String, data; header = hdr) - error(msg) - end - res + nargs = length(args) + if nargs == 1 + return args[1] + end + + tp = typeof(args[1]) + errs = [] + res = tp((resolve!(f, map(t -> getfield(t, f), args), errs) for f in fieldnames(tp))...) + + if length(errs) > 0 + msg = "Following fields have conflicting values:\n" + # filter fields with errors for all arguments + data = map(t -> map(f -> getfield(t, f), errs), args) |> collect |> x -> hcat(errs, x...) + hdr = ["fields", ((length(names) == 0) ? (1:length(args)) : names)...] + msg *= pretty_table(String, data; header = hdr) + error(msg) + end + res end # JSON parsing utility @@ -100,61 +100,61 @@ components of the reference. """ function json_get(json, reference::String; trunc::Int = 0) - function to_idx(token) - v = split(token, ".") - # JSON is 0-indexed, Julia is 1-indexed - length(v) > 1 ? [Symbol(v[1]), 1 + parse(Int, v[2])] : [Symbol(v[1])] - end - # NOTE: index 2:end because there is a leading '/' - idx = collect(Iterators.flatten(map(to_idx, split(reference, "/@"))))[2:(end-trunc)] - reduce(getindex, idx; init = json) # since $ref is from JSON, assume valid + function to_idx(token) + v = split(token, ".") + # JSON is 0-indexed, Julia is 1-indexed + length(v) > 1 ? [Symbol(v[1]), 1 + parse(Int, v[2])] : [Symbol(v[1])] + end + # NOTE: index 2:end because there is a leading '/' + idx = collect(Iterators.flatten(map(to_idx, split(reference, "/@"))))[2:(end - trunc)] + reduce(getindex, idx; init = json) # since $ref is from JSON, assume valid end # FIXME: ideally idcs should be typed Vector{Union{Int64,Symbol}} function json_get(json, idcs; default::Any = nothing) - reduce_unless((ret, i) -> get(ret, i, default), idcs; init = json, sentinel = default) + reduce_unless((ret, i) -> get(ret, i, default), idcs; init = json, sentinel = default) end # ESDL class parsers function cost_info(asset) - basepath = [:costInformation, :investmentCosts] - cost = json_get(asset, [:costInformation]; default = nothing) - if cost == nothing - return (nothing, nothing, nothing, nothing) - end - - ret = [] - for cost_type in (:investmentCosts, :variableOperationalAndMaintenanceCosts) - !in(cost_type, keys(cost)) && continue - val = json_get(cost, [cost_type, :value]; default = nothing) - unit = Dict( - k => json_get(asset, [basepath..., :profileQuantityAndUnit, k]; default = nothing) - for k in (:unit, :perMultiplier, :perUnit) - ) - push!(ret, val, "$(unit[:unit])/$(unit[:perMultiplier])$(unit[:perUnit])") - end - ret + basepath = [:costInformation, :investmentCosts] + cost = json_get(asset, [:costInformation]; default = nothing) + if cost == nothing + return (nothing, nothing, nothing, nothing) + end + + ret = [] + for cost_type in (:investmentCosts, :variableOperationalAndMaintenanceCosts) + !in(cost_type, keys(cost)) && continue + val = json_get(cost, [cost_type, :value]; default = nothing) + unit = Dict( + k => json_get(asset, [basepath..., :profileQuantityAndUnit, k]; default = nothing) for + k in (:unit, :perMultiplier, :perUnit) + ) + push!(ret, val, "$(unit[:unit])/$(unit[:perMultiplier])$(unit[:perUnit])") + end + ret end # struct to hold parsed data struct Asset - initial_capacity::Union{Float64,Nothing} - lifetime::Union{Float64,Nothing} - initial_storage_level::Union{Float64,Nothing} - investment_cost::Union{Float64,Nothing} - investment_cost_unit::Union{String,Nothing} - variable_cost::Union{Float64,Nothing} - variable_cost_unit::Union{String,Nothing} + initial_capacity::Union{Float64, Nothing} + lifetime::Union{Float64, Nothing} + initial_storage_level::Union{Float64, Nothing} + investment_cost::Union{Float64, Nothing} + investment_cost_unit::Union{String, Nothing} + variable_cost::Union{Float64, Nothing} + variable_cost_unit::Union{String, Nothing} end # constructor to call different parsers to determine the fields function Asset(asset::JSON3.Object) - Asset( - get(asset, :power, nothing), # initial_capacity - get(asset, :technicalLifetime, nothing), # lifetime - get(asset, :fillLevel, nothing), # initial_storage_level - cost_info(asset)..., # *_cost{,_unit} - ) + Asset( + get(asset, :power, nothing), # initial_capacity + get(asset, :technicalLifetime, nothing), # lifetime + get(asset, :fillLevel, nothing), # initial_storage_level + cost_info(asset)..., # *_cost{,_unit} + ) end # entry point for the parser @@ -171,8 +171,8 @@ the two nodes have conflicting asset values, an error is raised: """ function read_esdl_json(json_path) - json = JSON3.read(open(f -> read(f, String), json_path)) - flow_from_json(json) + json = JSON3.read(open(f -> read(f, String), json_path)) + flow_from_json(json) end """ @@ -185,36 +185,35 @@ by JSON3.jl): """ function flow_from_json(json) - """ - edge(from_asset, to_asset) - - Given a pair of assets, extract all the asset attributes and - construct an edge. - - """ - function edge(from_asset, to_asset) - names = [from_asset[:name], to_asset[:name]] - merged_asset = merge(Asset(from_asset), Asset(to_asset); names = names) - (names..., merged_asset) - end - - """ - edges(asset) - - Given an asset, find all this_asset -> other_asset edges - - """ - function edges(asset) - [ - edge(asset, json_get(json, to_port[Symbol("\$ref")]; trunc = 2)) for - port in asset[:port] if occursin("OutPort", port[:eClass]) for - to_port in port[:connectedTo] - ] - end - - flows = [] - flow_from_json_impl!(json, flows; find_edge = edges) - flows + """ + edge(from_asset, to_asset) + + Given a pair of assets, extract all the asset attributes and + construct an edge. + + """ + function edge(from_asset, to_asset) + names = [from_asset[:name], to_asset[:name]] + merged_asset = merge(Asset(from_asset), Asset(to_asset); names = names) + (names..., merged_asset) + end + + """ + edges(asset) + + Given an asset, find all this_asset -> other_asset edges + + """ + function edges(asset) + [ + edge(asset, json_get(json, to_port[Symbol("\$ref")]; trunc = 2)) for + port in asset[:port] if occursin("OutPort", port[:eClass]) for to_port in port[:connectedTo] + ] + end + + flows = [] + flow_from_json_impl!(json, flows; find_edge = edges) + flows end """ @@ -229,22 +228,22 @@ Find all flows (from/to node names) from a JSON document. """ function flow_from_json_impl!(json::JSON3.Object, flows; find_edge) - if :asset in keys(json) - flow_edges = [find_edge(asset) for asset in json[:asset] if :name in keys(asset)] - append!(flows, flow_edges...) - end - - if :area in keys(json) - flow_from_json_impl!(json[:area], flows; find_edge = find_edge) - end - - if :instance in keys(json) - flow_from_json_impl!(json[:instance], flows; find_edge = find_edge) - end + if :asset in keys(json) + flow_edges = [find_edge(asset) for asset in json[:asset] if :name in keys(asset)] + append!(flows, flow_edges...) + end + + if :area in keys(json) + flow_from_json_impl!(json[:area], flows; find_edge = find_edge) + end + + if :instance in keys(json) + flow_from_json_impl!(json[:instance], flows; find_edge = find_edge) + end end function flow_from_json_impl!(json::JSON3.Array{JSON3.Object}, flows; find_edge) - for json_el in json - flow_from_json_impl!(json_el, flows; find_edge = find_edge) - end + for json_el in json + flow_from_json_impl!(json_el, flows; find_edge = find_edge) + end end diff --git a/src/pipeline.jl b/src/pipeline.jl index cc6515c..369c391 100644 --- a/src/pipeline.jl +++ b/src/pipeline.jl @@ -9,59 +9,59 @@ export create_tbl, set_tbl_col _read_opts = pairs((header = true, skip = 1)) function check_file(source::String) - # FIXME: handle globs - isfile(source) + # FIXME: handle globs + isfile(source) end function check_tbl(con::DB, source::String) - df = DBInterface.execute(con, "SHOW TABLES") |> DF.DataFrame - source in df[!, :name] + df = DBInterface.execute(con, "SHOW TABLES") |> DF.DataFrame + source in df[!, :name] end function fmt_source(con::DB, source::String) - if check_tbl(con, source) - return source - elseif check_file(source) - return fmt_read(source; _read_opts...) - else - throw(NeitherTableNorFileError(con, source)) - end + if check_tbl(con, source) + return source + elseif check_file(source) + return fmt_read(source; _read_opts...) + else + throw(NeitherTableNorFileError(con, source)) + end end ## User facing functions below # TODO: prepared statements; not used for now struct Store - con::DB - read_csv::Stmt - - function Store(store::String) - con = DBInterface.connect(DB, store) - query = fmt_select(fmt_read("(?)"; _read_opts...)) - stmt = DBInterface.prepare(con, query) - new(con, stmt) - end + con::DB + read_csv::Stmt + + function Store(store::String) + con = DBInterface.connect(DB, store) + query = fmt_select(fmt_read("(?)"; _read_opts...)) + stmt = DBInterface.prepare(con, query) + new(con, stmt) + end end Store() = Store(":memory:") DEFAULT = Store() function tmp_tbl_name(source::String) - name, _ = splitext(basename(source)) - name = replace(name, r"[ ()\[\]{}\\+,.-]+" => "_") - "t_$(name)" + name, _ = splitext(basename(source)) + name = replace(name, r"[ ()\[\]{}\\+,.-]+" => "_") + "t_$(name)" end # TODO: support "CREATE OR REPLACE" & "IF NOT EXISTS" for all create_* functions function _create_tbl_impl(con::DB, query::String; name::String, tmp::Bool, show::Bool) - if length(name) > 0 - DBInterface.execute(con, "CREATE $(tmp ? "TEMP" : "") TABLE $name AS $query") - return show ? DF.DataFrame(DBInterface.execute(con, "SELECT * FROM $name")) : name - else # only show - res = DBInterface.execute(con, query) - return DF.DataFrame(res) - end + if length(name) > 0 + DBInterface.execute(con, "CREATE $(tmp ? "TEMP" : "") TABLE $name AS $query") + return show ? DF.DataFrame(DBInterface.execute(con, "SELECT * FROM $name")) : name + else # only show + res = DBInterface.execute(con, query) + return DF.DataFrame(res) + end end """ @@ -92,21 +92,21 @@ This also unconditionally sets the temporary table flag to `true`. """ function create_tbl( - con::DB, - source::String; - name::String = "", - tmp::Bool = false, - show::Bool = false, + con::DB, + source::String; + name::String = "", + tmp::Bool = false, + show::Bool = false, ) - check_file(source) ? true : throw(FileNotFoundError(source)) - query = fmt_select(fmt_read(source; _read_opts...)) + check_file(source) ? true : throw(FileNotFoundError(source)) + query = fmt_select(fmt_read(source; _read_opts...)) - if (length(name) == 0) && !show - tmp = true - name = tmp_tbl_name(source) - end + if (length(name) == 0) && !show + tmp = true + name = tmp_tbl_name(source) + end - return _create_tbl_impl(con, query; name = name, tmp = tmp, show = show) + return _create_tbl_impl(con, query; name = name, tmp = tmp, show = show) end """ @@ -153,51 +153,51 @@ alternative source. """ function create_tbl( - con::DB, - base_source::String, - alt_source::String; - on::Vector{Symbol}, - cols::Vector{Symbol}, - variant::String = "", - fill::Bool = true, - fill_values::Union{Missing,Dict} = missing, - tmp::Bool = false, - show::Bool = false, + con::DB, + base_source::String, + alt_source::String; + on::Vector{Symbol}, + cols::Vector{Symbol}, + variant::String = "", + fill::Bool = true, + fill_values::Union{Missing, Dict} = missing, + tmp::Bool = false, + show::Bool = false, ) - sources = [fmt_source(con, src) for src in (base_source, alt_source)] - query = fmt_join(sources...; on = on, cols = cols, fill = fill, fill_values = fill_values) + sources = [fmt_source(con, src) for src in (base_source, alt_source)] + query = fmt_join(sources...; on = on, cols = cols, fill = fill, fill_values = fill_values) - if (length(variant) == 0) && !show - tmp = true - variant = tmp_tbl_name(alt_source) - end + if (length(variant) == 0) && !show + tmp = true + variant = tmp_tbl_name(alt_source) + end - return _create_tbl_impl(con, query; name = variant, tmp = tmp, show = show) + return _create_tbl_impl(con, query; name = variant, tmp = tmp, show = show) end function _get_index(con::DB, source::String, on::Symbol) - # TODO: for file source instead of reading again, save to a tmp table - source = fmt_source(con, source) - base = DBInterface.execute(con, "SELECT $on FROM $source") |> DF.DataFrame - return getproperty(base, on) + # TODO: for file source instead of reading again, save to a tmp table + source = fmt_source(con, source) + base = DBInterface.execute(con, "SELECT $on FROM $source") |> DF.DataFrame + return getproperty(base, on) end function _set_tbl_col_impl( - con::DB, - source::String, - idx::Vector, - vals::Vector; - on::Symbol, - col::Symbol, - opts..., + con::DB, + source::String, + idx::Vector, + vals::Vector; + on::Symbol, + col::Symbol, + opts..., ) - df = DF.DataFrame([idx, vals], [on, col]) - tmp_tbl = "t_col_$(col)" - register_data_frame(con, df, tmp_tbl) - # FIXME: should be fill=error (currently not implemented) - res = create_tbl(con, source, tmp_tbl; on = [on], cols = [col], fill = false, opts...) - unregister_data_frame(con, tmp_tbl) - return res + df = DF.DataFrame([idx, vals], [on, col]) + tmp_tbl = "t_col_$(col)" + register_data_frame(con, df, tmp_tbl) + # FIXME: should be fill=error (currently not implemented) + res = create_tbl(con, source, tmp_tbl; on = [on], cols = [col], fill = false, opts...) + unregister_data_frame(con, tmp_tbl) + return res end """ @@ -223,48 +223,48 @@ All other options behave as the two source version of `create_tbl`. """ function set_tbl_col( - con::DB, - source::String, - cols::Dict{Symbol,Vector{T}}; - on::Symbol, - variant::String = "", - tmp::Bool = false, - show::Bool = false, -) where {T<:Union{Int64,Float64,String,Bool}} - # TODO: is it worth it to have the ability to set multiple - # columns? If such a feature is required, we can use - # cols::Dict{Symbol, Vector{Any}}, and get the cols and vals - # as: keys(cols), and values(cols) - - # for now, support only one column - if length(cols) > 1 - throw(DomainError(keys(cols), "only single column is support")) - end - - idx = _get_index(con, source, on) - vals = first(values(cols)) - if length(idx) != length(vals) - msg = "Length of index column and values are different\n" - _cols = [idx, vals] - data = - [get.(_cols, i, "-") for i = 1:maximum(length, _cols)] |> - Iterators.flatten |> - collect |> - x -> reshape(x, 2, :) |> permutedims - msg *= pretty_table(String, data; header = ["index", "value"]) - throw(DimensionMismatch(msg)) - end - _set_tbl_col_impl( - con, - source, - idx, - vals; - on = on, - col = first(keys(cols)), - variant = variant, - tmp = tmp, - show = show, - ) + con::DB, + source::String, + cols::Dict{Symbol, Vector{T}}; + on::Symbol, + variant::String = "", + tmp::Bool = false, + show::Bool = false, +) where {T <: Union{Int64, Float64, String, Bool}} + # TODO: is it worth it to have the ability to set multiple + # columns? If such a feature is required, we can use + # cols::Dict{Symbol, Vector{Any}}, and get the cols and vals + # as: keys(cols), and values(cols) + + # for now, support only one column + if length(cols) > 1 + throw(DomainError(keys(cols), "only single column is support")) + end + + idx = _get_index(con, source, on) + vals = first(values(cols)) + if length(idx) != length(vals) + msg = "Length of index column and values are different\n" + _cols = [idx, vals] + data = + [get.(_cols, i, "-") for i in 1:maximum(length, _cols)] |> + Iterators.flatten |> + collect |> + x -> reshape(x, 2, :) |> permutedims + msg *= pretty_table(String, data; header = ["index", "value"]) + throw(DimensionMismatch(msg)) + end + _set_tbl_col_impl( + con, + source, + idx, + vals; + on = on, + col = first(keys(cols)), + variant = variant, + tmp = tmp, + show = show, + ) end """ @@ -290,61 +290,61 @@ function. """ function set_tbl_col( - con::DB, - source::String, - cols::Dict{Symbol,T}; - on::Symbol, - where_::String = "", - variant::String = "", - tmp::Bool = false, - show::Bool = false, + con::DB, + source::String, + cols::Dict{Symbol, T}; + on::Symbol, + where_::String = "", + variant::String = "", + tmp::Bool = false, + show::Bool = false, ) where {T} - # FIXME: accept NamedTuple|Dict as cols in stead of value & col - source = fmt_source(con, source) - subquery = fmt_select(source; cols...) - if length(where_) > 0 - subquery *= " WHERE $(where_)" - end - - # FIXME: resolve String|Symbol schizophrenic API - query = fmt_join(source, "($subquery)"; on = [on], cols = [keys(cols)...], fill = true) - - if (length(variant) == 0) && !show - tmp = true - variant = tmp_tbl_name(source) - end - - return _create_tbl_impl(con, query; name = variant, tmp = tmp, show = show) + # FIXME: accept NamedTuple|Dict as cols in stead of value & col + source = fmt_source(con, source) + subquery = fmt_select(source; cols...) + if length(where_) > 0 + subquery *= " WHERE $(where_)" + end + + # FIXME: resolve String|Symbol schizophrenic API + query = fmt_join(source, "($subquery)"; on = [on], cols = [keys(cols)...], fill = true) + + if (length(variant) == 0) && !show + tmp = true + variant = tmp_tbl_name(source) + end + + return _create_tbl_impl(con, query; name = variant, tmp = tmp, show = show) end function set_tbl_col( - con::DB, - source::String; - on::Symbol, - col::Symbol, - apply::Function, - variant::String = "", - tmp::Bool = false, - show::Bool = false, + con::DB, + source::String; + on::Symbol, + col::Symbol, + apply::Function, + variant::String = "", + tmp::Bool = false, + show::Bool = false, ) end function select( - con::DB, - source::String, - expression::String; - name::String = "", - tmp::Bool = false, - show::Bool = false, + con::DB, + source::String, + expression::String; + name::String = "", + tmp::Bool = false, + show::Bool = false, ) - src = fmt_source(con, source) - query = "SELECT * FROM $src WHERE $expression" + src = fmt_source(con, source) + query = "SELECT * FROM $src WHERE $expression" - if (length(name) == 0) && !show - tmp = true - name = tmp_tbl_name(source) - end + if (length(name) == 0) && !show + tmp = true + name = tmp_tbl_name(source) + end - return _create_tbl_impl(con, query; name = name, tmp = tmp, show = show) + return _create_tbl_impl(con, query; name = name, tmp = tmp, show = show) end # TODO: diff --git a/test/runtests.jl b/test/runtests.jl index 1155dab..1fea2b7 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,4 +1,4 @@ -import TulipaIO +using TulipaIO: TulipaIO import Test: @test, @testset, @test_throws @@ -7,5 +7,5 @@ const DATA = joinpath(@__DIR__, "data") # Run all files in test folder starting with `test-` and ending with `.jl` test_files = filter(file -> startswith("test-")(file) && endswith(".jl")(file), readdir(@__DIR__)) for file in test_files - include(file) + include(file) end diff --git a/test/test-parsers.jl b/test/test-parsers.jl index 965b5a8..e0c826c 100644 --- a/test/test-parsers.jl +++ b/test/test-parsers.jl @@ -1,67 +1,67 @@ -import JSON3 +using JSON3: JSON3 @testset "Parsing utilities" begin - @testset "reduce_unless" begin - _sum = (i, j) -> nothing in (i, j) ? nothing : i + j - res = TulipaIO.reduce_unless(_sum, 1:3; init = 0, sentinel = nothing) - @test res == sum(1:3) - res = TulipaIO.reduce_unless(_sum, [1:3..., nothing]; init = 0, sentinel = nothing) - @test res == nothing - end + @testset "reduce_unless" begin + _sum = (i, j) -> nothing in (i, j) ? nothing : i + j + res = TulipaIO.reduce_unless(_sum, 1:3; init = 0, sentinel = nothing) + @test res == sum(1:3) + res = TulipaIO.reduce_unless(_sum, [1:3..., nothing]; init = 0, sentinel = nothing) + @test res == nothing + end - @testset "resolve!" begin - errs = [] - TulipaIO.resolve!(:fail, [ones(Int, 3)..., nothing], errs) - @test length(errs) == 0 - TulipaIO.resolve!(:fail, [ones(Int, 3)..., 2, nothing], errs) - @test :fail in errs - TulipaIO.resolve!("fail", [ones(Int, 3)..., 2, nothing], errs) - @test "fail" in errs - end + @testset "resolve!" begin + errs = [] + TulipaIO.resolve!(:fail, [ones(Int, 3)..., nothing], errs) + @test length(errs) == 0 + TulipaIO.resolve!(:fail, [ones(Int, 3)..., 2, nothing], errs) + @test :fail in errs + TulipaIO.resolve!("fail", [ones(Int, 3)..., 2, nothing], errs) + @test "fail" in errs + end - @testset "merge" begin - struct Data - foo::Union{Int,Nothing} - bar::Union{Bool,Nothing} - baz::Union{String,Nothing} - end - d0 = Data(nothing, nothing, nothing) - d1 = Data(42, true, "answer") - d2 = Data(42, true, nothing) - d3 = Data(42, false, "not answer") - d4 = Data(99, false, "not answer") + @testset "merge" begin + struct Data + foo::Union{Int, Nothing} + bar::Union{Bool, Nothing} + baz::Union{String, Nothing} + end + d0 = Data(nothing, nothing, nothing) + d1 = Data(42, true, "answer") + d2 = Data(42, true, nothing) + d3 = Data(42, false, "not answer") + d4 = Data(99, false, "not answer") - # nothing is overridden by a value - res = merge(d0, d1) - @test res == d1 - res = merge(d1, d2) - @test res == d1 - @test_throws ErrorException merge(d1, d3) # error when conflicting values - if (VERSION.major >= 1) && (VERSION.minor >= 8) - # error message lists fields w/ conflicting values - @test_throws r"bar.+\n.+baz" merge(d1, d2, d3) - @test_throws r"foo.+\n.+bar.+\n.+baz" merge(d1, d2, d4) - # error message column names - @test_throws r"fields.+1.+2" merge(d2, d4) # default: sequence - @test_throws r"fields.+bla.+dibla" merge(d2, d4; names = ["bla", "dibla"]) - end + # nothing is overridden by a value + res = merge(d0, d1) + @test res == d1 + res = merge(d1, d2) + @test res == d1 + @test_throws ErrorException merge(d1, d3) # error when conflicting values + if (VERSION.major >= 1) && (VERSION.minor >= 8) + # error message lists fields w/ conflicting values + @test_throws r"bar.+\n.+baz" merge(d1, d2, d3) + @test_throws r"foo.+\n.+bar.+\n.+baz" merge(d1, d2, d4) + # error message column names + @test_throws r"fields.+1.+2" merge(d2, d4) # default: sequence + @test_throws r"fields.+bla.+dibla" merge(d2, d4; names = ["bla", "dibla"]) end + end end @testset "Follow references in JSON" begin - json_path = joinpath(DATA, "esdl/norse-mythology.json") - json = JSON3.read(open(f -> read(f, String), json_path)) - @testset "String references" begin - target = TulipaIO.json_get(json, "//@instance.0/@area/@area.1/@asset.1/@port.1") - @test "ca13a453-57d1-4a63-b933-ca63fe33af34" == target[:id] - target = TulipaIO.json_get(json, "//@instance.0/@area/@area.1/@asset.1/@port.1"; trunc = 2) - @test "GasNetwork_6912" == target[:name] - end + json_path = joinpath(DATA, "esdl/norse-mythology.json") + json = JSON3.read(open(f -> read(f, String), json_path)) + @testset "String references" begin + target = TulipaIO.json_get(json, "//@instance.0/@area/@area.1/@asset.1/@port.1") + @test "ca13a453-57d1-4a63-b933-ca63fe33af34" == target[:id] + target = TulipaIO.json_get(json, "//@instance.0/@area/@area.1/@asset.1/@port.1"; trunc = 2) + @test "GasNetwork_6912" == target[:name] + end - @testset "Key/index references" begin - # NOTE: "//@instance.0/@area/@area.1/@area.0/@asset.4", indices are 1-indexed in Julia - target = TulipaIO.json_get(json, [:instance, 1, :area, :area, 2, :area, 1, :asset, 5]) - @test "PumpedHydroPower_eabf" == target[:name] - @test :costInformation in keys(target) - end + @testset "Key/index references" begin + # NOTE: "//@instance.0/@area/@area.1/@area.0/@asset.4", indices are 1-indexed in Julia + target = TulipaIO.json_get(json, [:instance, 1, :area, :area, 2, :area, 1, :asset, 5]) + @test "PumpedHydroPower_eabf" == target[:name] + @test :costInformation in keys(target) + end end diff --git a/test/test-pipeline.jl b/test/test-pipeline.jl index b7e9f54..585282d 100644 --- a/test/test-pipeline.jl +++ b/test/test-pipeline.jl @@ -1,16 +1,16 @@ -import CSV +using CSV: CSV import DataFrames as DF import DuckDB: DB, DBInterface TIO = TulipaIO function shape(df::DF.DataFrame) - return (DF.nrow(df), DF.ncol(df)) + return (DF.nrow(df), DF.ncol(df)) end function tmp_tbls(con::DB) - res = DBInterface.execute(con, "SELECT name FROM (SHOW ALL TABLES) WHERE temporary = true") - return DF.DataFrame(res) + res = DBInterface.execute(con, "SELECT name FROM (SHOW ALL TABLES) WHERE temporary = true") + return DF.DataFrame(res) end """ @@ -22,229 +22,216 @@ is returned. It uniquifies columns with clashing names (see `?DF.leftjoin`), and stores a "source" under the `:source` column. """ -function join_cmp(df1, df2, cols; on::Union{Symbol,Vector{Symbol}}) - DF.leftjoin(df1[!, cols], df2[!, cols]; on = on, makeunique = true, source = :source) +function join_cmp(df1, df2, cols; on::Union{Symbol, Vector{Symbol}}) + DF.leftjoin(df1[!, cols], df2[!, cols]; on = on, makeunique = true, source = :source) end @testset "Utilities" begin - csv_path = joinpath(DATA, "Norse/assets-data.csv") - - # redundant for the current implementation, needed when we support globs - @test TIO.check_file(csv_path) - @test !TIO.check_file("not-there") - - con = DBInterface.connect(DB) - tbl_name = "mytbl" - - @testset "Check if table exists" begin - DBInterface.execute(con, "CREATE TABLE $tbl_name AS SELECT * FROM range(5)") - @test TIO.check_tbl(con, tbl_name) - @test !TIO.check_tbl(con, "not_there") - end - - @testset "Conditionally format source as SQL" begin - read_ = TIO.fmt_source(con, csv_path) - @test occursin("read_csv", read_) - @test occursin(csv_path, read_) - @test TIO.fmt_source(con, tbl_name) == tbl_name - @test_throws TIO.NeitherTableNorFileError TIO.fmt_source(con, "not-there") - if (VERSION.major >= 1) && (VERSION.minor >= 8) - msg_re = r"not-there.+" - msg_re *= "$con" - @test_throws msg_re TIO.fmt_source(con, "not-there") - end + csv_path = joinpath(DATA, "Norse/assets-data.csv") + + # redundant for the current implementation, needed when we support globs + @test TIO.check_file(csv_path) + @test !TIO.check_file("not-there") + + con = DBInterface.connect(DB) + tbl_name = "mytbl" + + @testset "Check if table exists" begin + DBInterface.execute(con, "CREATE TABLE $tbl_name AS SELECT * FROM range(5)") + @test TIO.check_tbl(con, tbl_name) + @test !TIO.check_tbl(con, "not_there") + end + + @testset "Conditionally format source as SQL" begin + read_ = TIO.fmt_source(con, csv_path) + @test occursin("read_csv", read_) + @test occursin(csv_path, read_) + @test TIO.fmt_source(con, tbl_name) == tbl_name + @test_throws TIO.NeitherTableNorFileError TIO.fmt_source(con, "not-there") + if (VERSION.major >= 1) && (VERSION.minor >= 8) + msg_re = r"not-there.+" + msg_re *= "$con" + @test_throws msg_re TIO.fmt_source(con, "not-there") end + end end @testset "Read CSV" begin - csv_path = joinpath(DATA, "Norse/assets-data.csv") - csv_copy = replace(csv_path, "data.csv" => "data-copy.csv") - csv_fill = replace(csv_path, "data.csv" => "data-alt.csv") + csv_path = joinpath(DATA, "Norse/assets-data.csv") + csv_copy = replace(csv_path, "data.csv" => "data-copy.csv") + csv_fill = replace(csv_path, "data.csv" => "data-alt.csv") - df_org = DF.DataFrame(CSV.File(csv_path; header = 2)) + df_org = DF.DataFrame(CSV.File(csv_path; header = 2)) - con = DBInterface.connect(DB) + con = DBInterface.connect(DB) - @testset "CSV -> DataFrame" begin - df_res = TIO.create_tbl(con, csv_path; show = true) - @test shape(df_org) == shape(df_res) - @test_throws TIO.FileNotFoundError TIO.create_tbl(con, "not-there") - if (VERSION.major >= 1) && (VERSION.minor >= 8) - @test_throws r"not-there" TIO.create_tbl(con, "not-there") - end + @testset "CSV -> DataFrame" begin + df_res = TIO.create_tbl(con, csv_path; show = true) + @test shape(df_org) == shape(df_res) + @test_throws TIO.FileNotFoundError TIO.create_tbl(con, "not-there") + if (VERSION.major >= 1) && (VERSION.minor >= 8) + @test_throws r"not-there" TIO.create_tbl(con, "not-there") end - - @testset "CSV w/ alternatives -> DataFrame" begin - opts = Dict(:on => [:name], :cols => [:investable], :show => true) - df_res = TIO.create_tbl(con, csv_path, csv_copy; opts..., fill = false) - df_exp = DF.DataFrame(CSV.File(csv_copy; header = 2)) - @test df_exp.investable == df_res.investable - @test df_org.investable != df_res.investable - - @testset "no filling for missing rows" begin - df_res = TIO.create_tbl(con, csv_path, csv_fill; opts..., fill = false) - df_ref = DF.DataFrame(CSV.File(csv_fill; header = 2)) - # NOTE: row order is different, join to determine equality - cmp = join_cmp(df_res, df_ref, ["name", "investable"]; on = :name) - @test (DF.subset(cmp, :investable_1 => DF.ByRow(ismissing)).source .== "left_only") |> - all - @test (DF.subset(cmp, :investable_1 => DF.ByRow(!ismissing)).source .== "both") |> all - end - - @testset "back-filling missing rows" begin - df_res = TIO.create_tbl(con, csv_path, csv_fill; opts..., fill = true) - df_exp = DF.DataFrame(CSV.File(csv_copy; header = 2)) - cmp = join_cmp(df_exp, df_res, ["name", "investable"]; on = :name) - @test all(cmp.investable .== cmp.investable_1) - @test (cmp.source .== "both") |> all - end - - @testset "back-filling missing rows w/ alternate values" begin - df_res = TIO.create_tbl( - con, - csv_path, - csv_fill; - opts..., - fill = true, - fill_values = Dict(:investable => true), - ) - df_ref = DF.DataFrame(CSV.File(csv_fill; header = 2)) - cmp = join_cmp(df_res, df_ref, ["name", "investable"]; on = :name) - @test (DF.subset(cmp, :investable_1 => DF.ByRow(ismissing)).investable) |> all - end + end + + @testset "CSV w/ alternatives -> DataFrame" begin + opts = Dict(:on => [:name], :cols => [:investable], :show => true) + df_res = TIO.create_tbl(con, csv_path, csv_copy; opts..., fill = false) + df_exp = DF.DataFrame(CSV.File(csv_copy; header = 2)) + @test df_exp.investable == df_res.investable + @test df_org.investable != df_res.investable + + @testset "no filling for missing rows" begin + df_res = TIO.create_tbl(con, csv_path, csv_fill; opts..., fill = false) + df_ref = DF.DataFrame(CSV.File(csv_fill; header = 2)) + # NOTE: row order is different, join to determine equality + cmp = join_cmp(df_res, df_ref, ["name", "investable"]; on = :name) + @test (DF.subset(cmp, :investable_1 => DF.ByRow(ismissing)).source .== "left_only") |> all + @test (DF.subset(cmp, :investable_1 => DF.ByRow(!ismissing)).source .== "both") |> all end - @testset "CSV -> table" begin - tbl_name = TIO.create_tbl(con, csv_path; name = "no_assets") - df_res = DF.DataFrame(DBInterface.execute(con, "SELECT * FROM $tbl_name")) - @test shape(df_org) == shape(df_res) - # @show df_org[1:3, 1:5] df_res[1:3, 1:5] - # - # FIXME: cannot do an equality check b/c CSV.File above over - # specifies column types: - # - # Row │ name type active - # │ String31 String15 Bool - # ─────┼─────────────────────────────────── - # 1 │ Asgard_Battery storage true - # - # instead of: - # - # Row │ name type active - # │ String? String? Bool? - # ─────┼─────────────────────────────────── - # 1 │ Asgard_Battery storage true - - @testset "temporary tables" begin - tbl_name = TIO.create_tbl(con, csv_path; name = "tmp_assets", tmp = true) - @test tbl_name in tmp_tbls(con)[!, :name] - - tbl_name = TIO.create_tbl(con, csv_path) - @test tbl_name in tmp_tbls(con)[!, :name] - @test tbl_name == "t_assets_data" # t_ - end + @testset "back-filling missing rows" begin + df_res = TIO.create_tbl(con, csv_path, csv_fill; opts..., fill = true) + df_exp = DF.DataFrame(CSV.File(csv_copy; header = 2)) + cmp = join_cmp(df_exp, df_res, ["name", "investable"]; on = :name) + @test all(cmp.investable .== cmp.investable_1) + @test (cmp.source .== "both") |> all end - @testset "table + CSV w/ alternatives -> table" begin - opts = Dict(:on => [:name], :cols => [:investable]) - tbl_name = TIO.create_tbl( - con, - "no_assets", - csv_copy; - variant = "alt_assets", - opts..., - fill = false, - ) - df_res = DF.DataFrame(DBInterface.execute(con, "SELECT * FROM $tbl_name")) - df_exp = DF.DataFrame(CSV.File(csv_copy; header = 2)) - @test df_exp.investable == df_res.investable - @test df_org.investable != df_res.investable - - @testset "temporary tables" begin - tbl_name = TIO.create_tbl(con, "no_assets", csv_copy; opts...) - @test tbl_name in tmp_tbls(con)[!, :name] - @test tbl_name == "t_assets_data_copy" # t_ - end - - @testset "back-filling missing rows" begin - tbl_name = TIO.create_tbl( - con, - "no_assets", - csv_fill; - variant = "alt_assets_filled", - opts..., - fill = true, - ) - df_res = DF.DataFrame(DBInterface.execute(con, "SELECT * FROM $tbl_name")) - df_exp = DF.DataFrame(CSV.File(csv_copy; header = 2)) - # NOTE: row order is different, join to determine equality - cmp = join_cmp(df_exp, df_res, ["name", "investable"]; on = :name) - @test all(cmp.investable .== cmp.investable_1) - @test (cmp.source .== "both") |> all - end - - @testset "back-filling missing rows w/ alternate values" begin - tbl_name = TIO.create_tbl( - con, - "no_assets", - csv_fill; - variant = "alt_assets_filled_alt", - opts..., - fill = true, - fill_values = Dict(:investable => true), - ) - df_res = DF.DataFrame(DBInterface.execute(con, "SELECT * FROM $tbl_name")) - df_ref = DF.DataFrame(CSV.File(csv_fill; header = 2)) - cmp = join_cmp(df_res, df_ref, ["name", "investable"]; on = :name) - @test (DF.subset(cmp, :investable_1 => DF.ByRow(ismissing)).investable) |> all - end + @testset "back-filling missing rows w/ alternate values" begin + df_res = TIO.create_tbl( + con, + csv_path, + csv_fill; + opts..., + fill = true, + fill_values = Dict(:investable => true), + ) + df_ref = DF.DataFrame(CSV.File(csv_fill; header = 2)) + cmp = join_cmp(df_res, df_ref, ["name", "investable"]; on = :name) + @test (DF.subset(cmp, :investable_1 => DF.ByRow(ismissing)).investable) |> all + end + end + + @testset "CSV -> table" begin + tbl_name = TIO.create_tbl(con, csv_path; name = "no_assets") + df_res = DF.DataFrame(DBInterface.execute(con, "SELECT * FROM $tbl_name")) + @test shape(df_org) == shape(df_res) + # @show df_org[1:3, 1:5] df_res[1:3, 1:5] + # + # FIXME: cannot do an equality check b/c CSV.File above over + # specifies column types: + # + # Row │ name type active + # │ String31 String15 Bool + # ─────┼─────────────────────────────────── + # 1 │ Asgard_Battery storage true + # + # instead of: + # + # Row │ name type active + # │ String? String? Bool? + # ─────┼─────────────────────────────────── + # 1 │ Asgard_Battery storage true + + @testset "temporary tables" begin + tbl_name = TIO.create_tbl(con, csv_path; name = "tmp_assets", tmp = true) + @test tbl_name in tmp_tbls(con)[!, :name] + + tbl_name = TIO.create_tbl(con, csv_path) + @test tbl_name in tmp_tbls(con)[!, :name] + @test tbl_name == "t_assets_data" # t_ + end + end + + @testset "table + CSV w/ alternatives -> table" begin + opts = Dict(:on => [:name], :cols => [:investable]) + tbl_name = + TIO.create_tbl(con, "no_assets", csv_copy; variant = "alt_assets", opts..., fill = false) + df_res = DF.DataFrame(DBInterface.execute(con, "SELECT * FROM $tbl_name")) + df_exp = DF.DataFrame(CSV.File(csv_copy; header = 2)) + @test df_exp.investable == df_res.investable + @test df_org.investable != df_res.investable + + @testset "temporary tables" begin + tbl_name = TIO.create_tbl(con, "no_assets", csv_copy; opts...) + @test tbl_name in tmp_tbls(con)[!, :name] + @test tbl_name == "t_assets_data_copy" # t_ end -end -@testset "Set table column" begin - csv_path = joinpath(DATA, "Norse/assets-data.csv") - csv_copy = replace(csv_path, "data.csv" => "data-copy.csv") - csv_fill = replace(csv_path, "data.csv" => "data-alt.csv") - - df_org = DF.DataFrame(CSV.File(csv_path; header = 2)) - - con = DBInterface.connect(DB) - - opts = Dict(:on => :name, :show => true) - @testset "w/ vector" begin - df_exp = DF.DataFrame(CSV.File(csv_copy; header = 2)) - df_res = TIO.set_tbl_col(con, csv_path, Dict(:investable => df_exp.investable); opts...) - # NOTE: row order is different, join to determine equality - cmp = join_cmp(df_exp, df_res, ["name", "investable"]; on = :name) - investable = cmp[!, [c for c in propertynames(cmp) if occursin("investable", String(c))]] - @test isequal.(investable[!, 1], investable[!, 2]) |> all - - # stupid Julia! grow up! - args = [con, csv_path, Dict(:investable => df_exp.investable[2:end])] - @test_throws DimensionMismatch TIO.set_tbl_col(args...; opts...) - if (VERSION.major >= 1) && (VERSION.minor >= 8) - @test_throws r"Length.+different" TIO.set_tbl_col(args...; opts...) - @test_throws r"index.+value" TIO.set_tbl_col(args...; opts...) - end + @testset "back-filling missing rows" begin + tbl_name = TIO.create_tbl( + con, + "no_assets", + csv_fill; + variant = "alt_assets_filled", + opts..., + fill = true, + ) + df_res = DF.DataFrame(DBInterface.execute(con, "SELECT * FROM $tbl_name")) + df_exp = DF.DataFrame(CSV.File(csv_copy; header = 2)) + # NOTE: row order is different, join to determine equality + cmp = join_cmp(df_exp, df_res, ["name", "investable"]; on = :name) + @test all(cmp.investable .== cmp.investable_1) + @test (cmp.source .== "both") |> all end - @testset "w/ constant" begin - df_res = TIO.set_tbl_col(con, csv_path, Dict(:investable => true); opts...) - @test df_res.investable |> all + @testset "back-filling missing rows w/ alternate values" begin + tbl_name = TIO.create_tbl( + con, + "no_assets", + csv_fill; + variant = "alt_assets_filled_alt", + opts..., + fill = true, + fill_values = Dict(:investable => true), + ) + df_res = DF.DataFrame(DBInterface.execute(con, "SELECT * FROM $tbl_name")) + df_ref = DF.DataFrame(CSV.File(csv_fill; header = 2)) + cmp = join_cmp(df_res, df_ref, ["name", "investable"]; on = :name) + @test (DF.subset(cmp, :investable_1 => DF.ByRow(ismissing)).investable) |> all end + end +end - @testset "w/ constant after filtering" begin - where_clause = TIO.FmtSQL.@where_(lifetime in 25:50, name % "Valhalla_%") - df_res = TIO.set_tbl_col( - con, - csv_path, - Dict(:investable => true); - opts..., - where_ = where_clause, - ) - @test shape(df_res) == shape(df_org) - df_res = - filter(row -> 25 <= row.lifetime <= 50 && startswith(row.name, "Valhalla_"), df_res) - @test df_res.investable |> all +@testset "Set table column" begin + csv_path = joinpath(DATA, "Norse/assets-data.csv") + csv_copy = replace(csv_path, "data.csv" => "data-copy.csv") + csv_fill = replace(csv_path, "data.csv" => "data-alt.csv") + + df_org = DF.DataFrame(CSV.File(csv_path; header = 2)) + + con = DBInterface.connect(DB) + + opts = Dict(:on => :name, :show => true) + @testset "w/ vector" begin + df_exp = DF.DataFrame(CSV.File(csv_copy; header = 2)) + df_res = TIO.set_tbl_col(con, csv_path, Dict(:investable => df_exp.investable); opts...) + # NOTE: row order is different, join to determine equality + cmp = join_cmp(df_exp, df_res, ["name", "investable"]; on = :name) + investable = cmp[!, [c for c in propertynames(cmp) if occursin("investable", String(c))]] + @test isequal.(investable[!, 1], investable[!, 2]) |> all + + # stupid Julia! grow up! + args = [con, csv_path, Dict(:investable => df_exp.investable[2:end])] + @test_throws DimensionMismatch TIO.set_tbl_col(args...; opts...) + if (VERSION.major >= 1) && (VERSION.minor >= 8) + @test_throws r"Length.+different" TIO.set_tbl_col(args...; opts...) + @test_throws r"index.+value" TIO.set_tbl_col(args...; opts...) end + end + + @testset "w/ constant" begin + df_res = TIO.set_tbl_col(con, csv_path, Dict(:investable => true); opts...) + @test df_res.investable |> all + end + + @testset "w/ constant after filtering" begin + where_clause = TIO.FmtSQL.@where_(lifetime in 25:50, name % "Valhalla_%") + df_res = + TIO.set_tbl_col(con, csv_path, Dict(:investable => true); opts..., where_ = where_clause) + @test shape(df_res) == shape(df_org) + df_res = filter(row -> 25 <= row.lifetime <= 50 && startswith(row.name, "Valhalla_"), df_res) + @test df_res.investable |> all + end end