Skip to content

Commit

Permalink
pre-commit run -a
Browse files Browse the repository at this point in the history
  • Loading branch information
@suvayu
suvayu committed Feb 13, 2024
1 parent d500619 commit afd641f
Show file tree
Hide file tree
Showing 5 changed files with 184 additions and 191 deletions.
108 changes: 54 additions & 54 deletions src/parsers.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,19 +11,19 @@ from/to node names:
[(from_name, to_name), (..., ...), ...]
"""
function read_esdl_json(json_path)
json = JSON3.read(open(f -> read(f, String), json_path))
flow_from_json(json)
json = JSON3.read(open(f -> read(f, String), json_path))
flow_from_json(json)
end

# ESDL class parsers
function cost_info(asset)
basepath = "/@costInformation/@investmentCosts"
val = json_get(asset, "$basepath/@value")
unit = Dict(
k => json_get(asset, "$basepath/@profileQuantityAndUnit/@$k") for
k in (:unit, :perMultiplier, :perUnit)
)
(val, "$(unit[:unit])/$(unit[:perMultiplier])$(unit[:perUnit])")
basepath = "/@costInformation/@investmentCosts"
val = json_get(asset, "$basepath/@value")
unit = Dict(
k => json_get(asset, "$basepath/@profileQuantityAndUnit/@$k") for
k in (:unit, :perMultiplier, :perUnit)
)
(val, "$(unit[:unit])/$(unit[:perMultiplier])$(unit[:perUnit])")
end

"""
Expand All @@ -35,29 +35,29 @@ by JSON3.jl):
[(from_name, to_name), (..., ...), ...]
"""
function flow_from_json(json)
"""
edges(asset)
Given an asset, find all this_asset -> other_asset edges
"""
function edges(asset)
# TODO: return type: Vector{Tuple{String, String, <struct>}};
# we can do this only after we have an ESDL example with a
# complete set of Tulipa attributes
[
(
asset[:name],
json_get(json, to_port[Symbol("\$ref")]; trunc = 2)[:name],
cost_info(asset)...,
) for port in asset[:port] if occursin("OutPort", port[:eClass]) for
to_port in port[:connectedTo]
]
end

flows = []
flow_from_json_impl!(json, flows; find_edge = edges)
flows
"""
edges(asset)
Given an asset, find all this_asset -> other_asset edges
"""
function edges(asset)
# TODO: return type: Vector{Tuple{String, String, <struct>}};
# we can do this only after we have an ESDL example with a
# complete set of Tulipa attributes
[
(
asset[:name],
json_get(json, to_port[Symbol("\$ref")]; trunc = 2)[:name],
cost_info(asset)...,
) for port in asset[:port] if occursin("OutPort", port[:eClass]) for
to_port in port[:connectedTo]
]
end

flows = []
flow_from_json_impl!(json, flows; find_edge = edges)
flows
end

"""
Expand All @@ -68,13 +68,13 @@ Given a JSON document, find the object pointed to by the reference
components of the reference.
"""
function json_get(json, reference::String; trunc = 0)
function to_idx(token)
v = split(token, ".")
length(v) > 1 ? [Symbol(v[1]), 1 + parse(Int, v[2])] : [Symbol(v[1])]
end
# NOTE: index 2:end because there is a leading '/'
idx = collect(Iterators.flatten(map(to_idx, split(reference, "/@"))))[2:(end-trunc)]
reduce(getindex, idx; init = json)
function to_idx(token)
v = split(token, ".")
length(v) > 1 ? [Symbol(v[1]), 1 + parse(Int, v[2])] : [Symbol(v[1])]
end
# NOTE: index 2:end because there is a leading '/'
idx = collect(Iterators.flatten(map(to_idx, split(reference, "/@"))))[2:(end - trunc)]
reduce(getindex, idx; init = json)
end

"""
Expand All @@ -88,22 +88,22 @@ Find all flows (from/to node names) from a JSON document.
originating from an asset
"""
function flow_from_json_impl!(json::JSON3.Object, flows; find_edge)
if :asset in keys(json)
flow_edges = [find_edge(asset) for asset in json[:asset] if :name in keys(asset)]
append!(flows, flow_edges...)
end

if :area in keys(json)
flow_from_json_impl!(json[:area], flows; find_edge = find_edge)
end

if :instance in keys(json)
flow_from_json_impl!(json[:instance], flows; find_edge = find_edge)
end
if :asset in keys(json)
flow_edges = [find_edge(asset) for asset in json[:asset] if :name in keys(asset)]
append!(flows, flow_edges...)
end

if :area in keys(json)
flow_from_json_impl!(json[:area], flows; find_edge = find_edge)
end

if :instance in keys(json)
flow_from_json_impl!(json[:instance], flows; find_edge = find_edge)
end
end

function flow_from_json_impl!(json::JSON3.Array{JSON3.Object}, flows; find_edge)
for json_el in json
flow_from_json_impl!(json_el, flows; find_edge = find_edge)
end
for json_el in json
flow_from_json_impl!(json_el, flows; find_edge = find_edge)
end
end
140 changes: 67 additions & 73 deletions src/pipeline.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,121 +3,115 @@ import DuckDB: DB, DBInterface, Stmt
import Printf: format, Format

function sprintf(fmt::String, args...)
format(Format(fmt), args...)
format(Format(fmt), args...)
end

function fmt_opts(source::String; opts...)
_src = '?' in source ? "$source" : "'$(source)'"
join(["$(_src)"; [join(p, "=") for p in opts]], ", ")
_src = '?' in source ? "$source" : "'$(source)'"
join(["$(_src)"; [join(p, "=") for p in opts]], ", ")
end

function reader(source::String)
_, ext = splitext(source)
if ext in (".csv", ".parquet", ".json")
return "read_$(ext[2:end])_auto"
elseif '?' in source
# FIXME: how to support other file formats?
return "read_csv_auto"
else
error("$(ext[2:end]): unsupported input file '$(source)'")
end
_, ext = splitext(source)
if ext in (".csv", ".parquet", ".json")
return "read_$(ext[2:end])_auto"
elseif '?' in source
# FIXME: how to support other file formats?
return "read_csv_auto"
else
error("$(ext[2:end]): unsupported input file '$(source)'")
end
end

function fmt_query(source::String; opts...)
sprintf("SELECT * FROM %s(%s)", reader(source), fmt_opts(source; opts...))
sprintf("SELECT * FROM %s(%s)", reader(source), fmt_opts(source; opts...))
end

function fmt_join(
subquery1::String,
subquery2::String;
on::Vector{String},
cols::Vector{String},
)
exclude = join(cols, ", ")
include = join(map(c -> "t2.$c", cols), ", ")
select_ = "SELECT t1.* EXCLUDE ($exclude), $include"
function fmt_join(subquery1::String, subquery2::String; on::Vector{String}, cols::Vector{String})
exclude = join(cols, ", ")
include = join(map(c -> "t2.$c", cols), ", ")
select_ = "SELECT t1.* EXCLUDE ($exclude), $include"

join_on = join(map(c -> "t1.$c = t2.$c", on), " AND ")
from_ = "FROM $subquery1 t1 LEFT JOIN $subquery2 t2 ON ($join_on)"
join_on = join(map(c -> "t1.$c = t2.$c", on), " AND ")
from_ = "FROM $subquery1 t1 LEFT JOIN $subquery2 t2 ON ($join_on)"

"$(select_)\n$(from_)"
"$(select_)\n$(from_)"
end

function check_file(source::String)
# FIXME: handle globs
isfile(source) || throw(ArgumentError("$(source): is not a regular file"))
source
# FIXME: handle globs
isfile(source) || throw(ArgumentError("$(source): is not a regular file"))
source
end

struct Store
con::DB
read_csv::Stmt

function Store(store::String)
con = DBInterface.connect(DB, store)
query = fmt_query("(?)"; header = true, skip = 1)
stmt = DBInterface.prepare(con, query)
new(con, stmt)
end
con::DB
read_csv::Stmt

function Store(store::String)
con = DBInterface.connect(DB, store)
query = fmt_query("(?)"; header = true, skip = 1)
stmt = DBInterface.prepare(con, query)
new(con, stmt)
end
end

Store() = Store(":memory:")

function read_csv(con::DB, source::String)
check_file(source)
query = fmt_query("(?)"; header = true, skip = 1)
res = DBInterface.execute(con, query, [source])
return DF.DataFrame(res)
check_file(source)
query = fmt_query("(?)"; header = true, skip = 1)
res = DBInterface.execute(con, query, [source])
return DF.DataFrame(res)
end

function read_csv_alt_cols(
con::DB,
source1::String,
source2::String;
on::Vector{String},
cols::Vector{String},
con::DB,
source1::String,
source2::String;
on::Vector{String},
cols::Vector{String},
)
check_file(source1)
check_file(source2)
subquery = sprintf("read_csv_auto(%s)", fmt_opts("(?)"; header = true, skip = 1))
query = fmt_join(subquery, subquery, on = on, cols = cols)
res = DBInterface.execute(con, query, [source1, source2])
return DF.DataFrame(res)
check_file(source1)
check_file(source2)
subquery = sprintf("read_csv_auto(%s)", fmt_opts("(?)"; header = true, skip = 1))
query = fmt_join(subquery, subquery, on = on, cols = cols)
res = DBInterface.execute(con, query, [source1, source2])
return DF.DataFrame(res)
end

function tmp_tbl_name(source::String)
name, _ = replace(splitext(basename(source)), r"[ ()\[\]{}\\+,.]+" => "_")
"t_$(name)"
name, _ = replace(splitext(basename(source)), r"[ ()\[\]{}\\+,.]+" => "_")
"t_$(name)"
end

function create_tbl(con::DB, name::String, source::String; tmp::Bool = false)
check_file(source)
query = fmt_query(source; header = true, skip = 1)
TEMP = tmp ? "TEMP" : ""
DBInterface.execute(con, "CREATE $TEMP TABLE $name AS $query")
check_file(source)
query = fmt_query(source; header = true, skip = 1)
TEMP = tmp ? "TEMP" : ""
DBInterface.execute(con, "CREATE $TEMP TABLE $name AS $query")
end

function create_alt_tbl(
con::DB,
ref::String,
alt::String,
source::String;
on::Vector{String},
cols::Vector{String},
tmp::Bool = false,
con::DB,
ref::String,
alt::String,
source::String;
on::Vector{String},
cols::Vector{String},
tmp::Bool = false,
)
check_file(source)
TEMP = tmp ? "TEMP" : ""
# TODO: support "CREATE OR REPLACE" & "IF NOT EXISTS"
create_ = "CREATE $TEMP TABLE $alt AS"
check_file(source)
TEMP = tmp ? "TEMP" : ""
# TODO: support "CREATE OR REPLACE" & "IF NOT EXISTS"
create_ = "CREATE $TEMP TABLE $alt AS"

subquery = sprintf("%s(%s)", reader(source), fmt_opts(source; header = true, skip = 1))
query = fmt_join(ref, subquery; on = on, cols = cols)
subquery = sprintf("%s(%s)", reader(source), fmt_opts(source; header = true, skip = 1))
query = fmt_join(ref, subquery; on = on, cols = cols)

DBInterface.execute(con, "$(create_)\n$(query)")
DBInterface.execute(con, "$(create_)\n$(query)")
end


# TODO:
# - filter rows (where clause)
# - dataframe as data source
5 changes: 5 additions & 0 deletions src/utils.jl
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
import Printf: format, Format

function sprintf(fmt::String, args...)
format(Format(fmt), args...)
end
48 changes: 24 additions & 24 deletions test/data/Norse/assets-data-copy.csv
View file
Original file line number Diff line number Diff line change
@@ -1,24 +1,24 @@
,{producer;consumer;storage;hub;conversion},{true;false},{true;false},{true;false},kEUR/MWh,kEUR/MW/year,MW,MW,MW,MW,MW/unit,MW/unit,MW,MW,MWh,MWh,h,kEUR/MW/year,year,0
name,type,active,investable,investment_integer,variable_cost,investment_cost,investment_limit,capacity,initial_capacity,peak_demand,charging_capacity,discharging_capacity,initial_charging_capacity,initial_discharging_capacity,initial_storage_capacity,initial_storage_level,energy_to_power_ratio,fixed_cost,lifetime,efficiency
Asgard_Battery,storage,true,true,true,0.003,300,,10,0,0,1,1,0,0,0,,100,3,10,0
Asgard_Solar,producer,true,true,true,0.001,350,50000,100,0,0,0,0,0,0,0,0,0,8,25,0
Asgard_E_demand,consumer,true,false,false,0,0,0,0,0,65787.17792,0,0,0,0,0,0,0,0,0,0
Asgard_CCGT,conversion,true,false,true,0.0015,650,,500,0,0,0,0,0,0,0,0,0,10,30,0.6
G_imports,producer,true,false,false,0.08,0,0,0,75000,0,0,0,0,0,0,0,0,0,0,0
Midgard_Wind,producer,true,true,true,0.002,1300,80000,3,0,0,0,0,0,0,0,0,0,15,25,0
Midgard_Hydro,storage,true,true,false,0.0012,1600,0,0,250,0,3,3,0,0,4000,2000,0,16,50,0
Midgard_Nuclear_SMR,producer,true,true,false,0.015,6000,,150,1000,0,0,0,0,0,0,0,0,101,60,0
Midgard_E_imports,producer,true,true,false,0.02,0,0,0,500,0,0,0,0,0,0,0,0,0,0,0
Midgard_CCGT,conversion,true,true,true,0.0015,650,,500,0,0,0,0,0,0,0,0,0,10,30,0.6
Midgard_E_demand,consumer,true,false,false,0,0,0,0,0,19604.76443,0,0,0,0,0,0,0,0,0,0
Valhalla_H2_generator,conversion,true,true,true,0,479,,100,0,0,0,0,0,0,0,0,0,10,20,0.6
Valhalla_H2_storage,storage,true,false,true,0,0.1,,500,0,0,3,3,0,0,0,0,10000,0.005,50,0
Valhalla_H2_demand,consumer,true,false,false,0,0,0,0,0,745.735,0,0,0,0,0,0,0,0,0,0
Valhalla_Electrolyser,conversion,true,true,true,0,1260,,100,500,0,0,0,0,0,0,0,0,60,25,0.704
Valhalla_E_balance,hub,true,false,false,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
Valhalla_E_exports,consumer,true,false,false,0.09,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0
Valhalla_Fuel_cell,conversion,true,true,true,0,800,,100,0,0,0,0,0,0,0,0,0,40,25,0.5
Valhalla_Heat_pump,conversion,true,true,true,0,300,,100,0,0,0,0,0,0,0,0,0,15,20,4
Valhalla_Waste_heat,producer,true,true,false,0.0025,1450,,200,0,0,0,0,0,0,0,0,0,0,30,0
Valhalla_Heat_demand,consumer,true,false,false,0,0,0,0,0,3548.42445,0,0,0,0,0,0,0,0,0,0
Valhalla_GT,conversion,true,true,true,0.0015,400,100000,500,0,0,0,0,0,0,0,0,0,3.85,30,0.6
,{producer;consumer;storage;hub;conversion},{true;false},{true;false},{true;false},kEUR/MWh,kEUR/MW/year,MW,MW,MW,MW,MW/unit,MW/unit,MW,MW,MWh,MWh,h,kEUR/MW/year,year,0
name,type,active,investable,investment_integer,variable_cost,investment_cost,investment_limit,capacity,initial_capacity,peak_demand,charging_capacity,discharging_capacity,initial_charging_capacity,initial_discharging_capacity,initial_storage_capacity,initial_storage_level,energy_to_power_ratio,fixed_cost,lifetime,efficiency
Asgard_Battery,storage,true,true,true,0.003,300,,10,0,0,1,1,0,0,0,,100,3,10,0
Asgard_Solar,producer,true,true,true,0.001,350,50000,100,0,0,0,0,0,0,0,0,0,8,25,0
Asgard_E_demand,consumer,true,false,false,0,0,0,0,0,65787.17792,0,0,0,0,0,0,0,0,0,0
Asgard_CCGT,conversion,true,false,true,0.0015,650,,500,0,0,0,0,0,0,0,0,0,10,30,0.6
G_imports,producer,true,false,false,0.08,0,0,0,75000,0,0,0,0,0,0,0,0,0,0,0
Midgard_Wind,producer,true,true,true,0.002,1300,80000,3,0,0,0,0,0,0,0,0,0,15,25,0
Midgard_Hydro,storage,true,true,false,0.0012,1600,0,0,250,0,3,3,0,0,4000,2000,0,16,50,0
Midgard_Nuclear_SMR,producer,true,true,false,0.015,6000,,150,1000,0,0,0,0,0,0,0,0,101,60,0
Midgard_E_imports,producer,true,true,false,0.02,0,0,0,500,0,0,0,0,0,0,0,0,0,0,0
Midgard_CCGT,conversion,true,true,true,0.0015,650,,500,0,0,0,0,0,0,0,0,0,10,30,0.6
Midgard_E_demand,consumer,true,false,false,0,0,0,0,0,19604.76443,0,0,0,0,0,0,0,0,0,0
Valhalla_H2_generator,conversion,true,true,true,0,479,,100,0,0,0,0,0,0,0,0,0,10,20,0.6
Valhalla_H2_storage,storage,true,false,true,0,0.1,,500,0,0,3,3,0,0,0,0,10000,0.005,50,0
Valhalla_H2_demand,consumer,true,false,false,0,0,0,0,0,745.735,0,0,0,0,0,0,0,0,0,0
Valhalla_Electrolyser,conversion,true,true,true,0,1260,,100,500,0,0,0,0,0,0,0,0,60,25,0.704
Valhalla_E_balance,hub,true,false,false,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
Valhalla_E_exports,consumer,true,false,false,0.09,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0
Valhalla_Fuel_cell,conversion,true,true,true,0,800,,100,0,0,0,0,0,0,0,0,0,40,25,0.5
Valhalla_Heat_pump,conversion,true,true,true,0,300,,100,0,0,0,0,0,0,0,0,0,15,20,4
Valhalla_Waste_heat,producer,true,true,false,0.0025,1450,,200,0,0,0,0,0,0,0,0,0,0,30,0
Valhalla_Heat_demand,consumer,true,false,false,0,0,0,0,0,3548.42445,0,0,0,0,0,0,0,0,0,0
Valhalla_GT,conversion,true,true,true,0.0015,400,100000,500,0,0,0,0,0,0,0,0,0,3.85,30,0.6
Loading

0 comments on commit afd641f

Please sign in to comment.