-
Notifications
You must be signed in to change notification settings - Fork 2
/
Copy pathyour_make.sample
114 lines (98 loc) · 5.89 KB
/
your_make.sample
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
## This is an example makefile created to demonstrate how to create and use targets as part of the ETL process.
## extract, transform, load (ETL) is a three-phase process where data is extracted, transformed (cleaned, sanitized, scrubbed)
## and loaded into an output data container
## -------------- EXPORT BLOCK ------------------------
# configuration file
file := ${GEOCINT_WORK_DIRECTORY}/config.inc.sh
# Add an export here for each variable from the configuration file that you are going to use in the targets.
export SLACK_CHANNEL = $(shell sed -n -e '/^SLACK_CHANNEL/p' ${file} | cut -d "=" -f 2)
export SLACK_BOT_NAME = $(shell sed -n -e '/^SLACK_BOT_NAME/p' ${file} | cut -d "=" -f 2)
export SLACK_BOT_EMOJI = $(shell sed -n -e '/^SLACK_BOT_EMOJI/p' ${file} | cut -d "=" -f 2)
export SLACK_KEY = $(shell sed -n -e '/^SLACK_KEY/p' ${file} | cut -d "=" -f 2)
# these makefiles are stored in geocint-runner and geocint-openstreetmap repositories
# runner_make contains the basic set of targets for creating the project folder structure
# osm_make contains a set of targets for osm data processing
include runner_make osm_make
## ------------- CONTROL BLOCK -------------------------
# you can replace dev with the names of the final targets, that you will use to run the pipeline if you don't need all of them
# you can also add here the names of targets that should not be rebuilt automatically, just when conditions are met or at your request
# to do it just add these names after the colon separated by a space
all: dev ## [FINAL] Meta-target on top of all other targets, or targets on parking.
# by default the clean target is set to serve an update of the OpenStreetMap planet dump during every run
clean: ## [FINAL] Cleans the worktree for the next nightly run. Does not clean non-repeating targets.
if [ -f data/planet-is-broken ]; then rm -rf data/planet-latest.osm.pbf ; fi
rm -rf data/planet-is-broken
profile_make_clean data/planet-latest-updated.osm.pbf
## --------------- SAMPLE TARGET CHAIN ------------------
## --------------- DATA PROCESSING BLOCK ----------------
db/table/fire_stations: db/index/osm_tags_idx ## Extract centroids of fire stations from the osm table
# db/index/osm_tags_idx is a dependency from geocint-openstreetmap Makefile
# drop table if exist to avoid conflict
psql -c "drop table if exists fire_stations;"
# extract fire stations from the osm table to the new table
# you can move such a big query to a separate SQL file like tables/fire_station.sql
# and execute them with psql -f tables/fire_station.sql
# We run them from makefile just to keep the sample in one file
psql -c "create table fire_stations as ( \
select osm_id, \
tags ->> 'amenity' "type", \
ST_Centroid(geog::geometry) as geom \
from osm o \
where tags @> '{"amenity":"fire_station"}' \
order by osm_id \
);"
# update db/table/fire_stations time mark to be able to create the correct graph using make-profiler
# keep in mind that $@ at Makefile is an equivalent of the name of the target containing it
touch $@
db/table/fire_stations_h3_r8: db/table/fire_stations ## Create h3 8 resolution hexagons from centroids of fire stations
psql -c "drop table if exists fire_stations_h3_r8;"
# generate h3 8 resolution hexes indexes and boundaries from geometry using h3_pg
# for more information see https://github.com/bytesandbrains/h3-pg
psql -c "create table fire_stations_h3_r8 as ( \
select h3_lat_lng_to_cell(ST_Transform(geom, 4326)::point, 8) as h3, \
h3_cell_to_boundary_geometry(h3_lat_lng_to_cell(ST_Transform(geom, 4326)::point, 8)) as geom, \
osm_id, \
type \
from fire_stations \
);"
touch $@
db/table/fire_stations_h3_r8_count: db/table/fire_stations_h3_r8 ## Count the number of fire stations for each hexagon
psql -c "drop table if exists fire_stations_h3_r8_count;"
# Count the number of fire stations for each hexagon
psql -c "create table fire_stations_h3_r8_count as ( \
select count(h3) as fire_stations_number, \
h3, \
geom \
from fire_stations_h3_r8 \
group by 2,3 \
);"
touch $@
## ---------------- DATA EXPORT BLOCK ----------------------
data/out/fire_stations: | data/out ## Create a folder for data extraction
# data/out is a dependency from geocint-runner Makefile
mkdir -p $@
data/out/fire_stations/fire_stations_h3_r8_count.gpkg.gz: db/table/fire_stations_h3_r8_count | data/out/fire_stations ## Extract from the database to a compressed gpkg file
# remove file if it exists to avoid conflicts
rm -f $@
rm -f data/out/fire_stations/fire_stations_h3_r8_count.gpkg
# extract data to gpkg file
ogr2ogr -f GPKG data/out/fire_stations/fire_stations_h3_r8_count.gpkg \
PG:'dbname=gis' \
-sql "select fire_stations_number, h3, geom from fire_stations_h3_r8_count order by h3" \
-lco "SPATIAL_INDEX=NO"\
-nln fire_stations_h3
# compress the extracted file
cd data/out/fire_stations; pigz fire_stations_h3_r8_count.gpkg
## ---------------- DATA DEPLOY BLOCK ----------------------
deploy/s3/fire_stations: data/out/fire_stations/fire_stations_h3_r8_count.gpkg.gz | deploy/s3 ## deploy/s3 is a dependency from geocint-runner Makefile
aws s3 cp data/out/fire_stations/fire_stations_h3_r8_count.gpkg.gz \
s3://your_s3_bucket/fire_stations_h3_r8_count.gpkg.gz \
--profile your_s3_profile \
--acl public-read
touch $@
# A dev target is an example of a meta-target that depends on all chains of targets and allows you to run the entire pipeline with only one target running.
# Also, this target allows you to send messages to the slack channel in case the entire pipeline was successfully completed (or you can perform any other action)
dev: deploy/s3/fire_stations ## Send a message about the successful execution of the pipeline
#send dev target successfully build- message to a slack channel
echo "dev target successfully build" | python scripts/slack_message.py $$SLACK_CHANNEL ${SLACK_BOT_NAME} $$SLACK_BOT_EMOJI
touch $@