From f034feffcbee152806a3a6fd7cce45d96c70996c Mon Sep 17 00:00:00 2001 From: IritaSee Date: Thu, 13 Jan 2022 20:58:28 +0700 Subject: [PATCH 01/15] add extra cd --- kuwala/scripts/initialize_windows.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/kuwala/scripts/initialize_windows.sh b/kuwala/scripts/initialize_windows.sh index 3269c380..fe79eb07 100644 --- a/kuwala/scripts/initialize_windows.sh +++ b/kuwala/scripts/initialize_windows.sh @@ -1,12 +1,19 @@ mkdir windows sed 's/\r$//' initialize_all_components.sh > ./windows/initialize_all_components.sh sed 's/\r$//' build_cli.sh > ./windows/build_cli.sh +sed -i '1 a cd ..' ./windows/build_cli.sh sed 's/\r$//' build_jupyter_notebook.sh > ./windows/build_jupyter_notebook.sh +sed -i '1 a cd ..' ./windows/build_jupyter_notebook.sh sed 's/\r$//' build_neo4j.sh > ./windows/build_neo4j.sh +sed -i '1 a cd ..' ./windows/build_neo4j.sh sed 's/\r$//' create_zip_archive.sh > ./windows/create_zip_archive.sh +sed -i '1 a cd ..' ./windows/create_zip_archive.sh sed 's/\r$//' initialize_core_components.sh > ./windows/initialize_core_components.sh sed 's/\r$//' initialize_git_submodules.sh > ./windows/initialize_git_submodules.sh +sed -i '1 a cd ..' ./windows/initialize_git_submodules.sh sed 's/\r$//' run_cli.sh > ./windows/run_cli.sh sed 's/\r$//' run_jupyter_notebook.sh > ./windows/run_jupyter_notebook.sh +sed -i '1 a cd ..' ./windows/run_jupyter_notebook.sh sed 's/\r$//' stop_all_containers.sh > ./windows/stop_all_containers.sh -sed 's/\r$//' build_all_containers.sh > ./windows/build_all_containers.sh \ No newline at end of file +sed 's/\r$//' build_all_containers.sh > ./windows/build_all_containers.sh +sed -i '1 a cd ..' ./windows/build_all_containers.sh \ No newline at end of file From 4063d61e7e7018e70a7442ec356cab4154a682f7 Mon Sep 17 00:00:00 2001 From: IritaSee Date: Fri, 14 Jan 2022 09:26:29 +0700 Subject: [PATCH 02/15] add extra cd .. for run_cli --- kuwala/scripts/initialize_windows.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/kuwala/scripts/initialize_windows.sh b/kuwala/scripts/initialize_windows.sh index fe79eb07..b744af6e 100644 --- a/kuwala/scripts/initialize_windows.sh +++ b/kuwala/scripts/initialize_windows.sh @@ -12,6 +12,7 @@ sed 's/\r$//' initialize_core_components.sh > ./windows/initialize_core_componen sed 's/\r$//' initialize_git_submodules.sh > ./windows/initialize_git_submodules.sh sed -i '1 a cd ..' ./windows/initialize_git_submodules.sh sed 's/\r$//' run_cli.sh > ./windows/run_cli.sh +sed -i '1 a cd ..' ./windows/run_cli.sh sed 's/\r$//' run_jupyter_notebook.sh > ./windows/run_jupyter_notebook.sh sed -i '1 a cd ..' ./windows/run_jupyter_notebook.sh sed 's/\r$//' stop_all_containers.sh > ./windows/stop_all_containers.sh From 5e2fd82211ddffaec936bd448c56df9fd495539d Mon Sep 17 00:00:00 2001 From: IritaSee Date: Fri, 14 Jan 2022 18:21:23 +0700 Subject: [PATCH 03/15] delete initialize_windows --- kuwala/scripts/initialize_windows.sh | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 kuwala/scripts/initialize_windows.sh diff --git a/kuwala/scripts/initialize_windows.sh b/kuwala/scripts/initialize_windows.sh deleted file mode 100644 index b744af6e..00000000 --- a/kuwala/scripts/initialize_windows.sh +++ /dev/null @@ -1,20 +0,0 @@ -mkdir windows -sed 's/\r$//' initialize_all_components.sh > ./windows/initialize_all_components.sh -sed 's/\r$//' build_cli.sh > ./windows/build_cli.sh -sed -i '1 a cd ..' ./windows/build_cli.sh -sed 's/\r$//' build_jupyter_notebook.sh > ./windows/build_jupyter_notebook.sh -sed -i '1 a cd ..' ./windows/build_jupyter_notebook.sh -sed 's/\r$//' build_neo4j.sh > ./windows/build_neo4j.sh -sed -i '1 a cd ..' ./windows/build_neo4j.sh -sed 's/\r$//' create_zip_archive.sh > ./windows/create_zip_archive.sh -sed -i '1 a cd ..' ./windows/create_zip_archive.sh -sed 's/\r$//' initialize_core_components.sh > ./windows/initialize_core_components.sh -sed 's/\r$//' initialize_git_submodules.sh > ./windows/initialize_git_submodules.sh -sed -i '1 a cd ..' ./windows/initialize_git_submodules.sh -sed 's/\r$//' run_cli.sh > ./windows/run_cli.sh -sed -i '1 a cd ..' ./windows/run_cli.sh -sed 's/\r$//' run_jupyter_notebook.sh > ./windows/run_jupyter_notebook.sh -sed -i '1 a cd ..' ./windows/run_jupyter_notebook.sh -sed 's/\r$//' stop_all_containers.sh > ./windows/stop_all_containers.sh -sed 's/\r$//' build_all_containers.sh > ./windows/build_all_containers.sh -sed -i '1 a cd ..' ./windows/build_all_containers.sh \ No newline at end of file From b636e854fbf9507a78179b741d54087f15ac97c1 Mon Sep 17 00:00:00 2001 From: IritaSee Date: Fri, 14 Jan 2022 18:41:33 +0700 Subject: [PATCH 04/15] edit main readme for windows issue --- README.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 66f2957d..cd730241 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,9 @@ Uber. For the complete documentation please go to the [H3 Repo](https://github.c Installed version of *Python3*, *Docker* and *docker-compose* ([*Go here for instructions*](https://docs.docker.com/compose/install/)) -***Note***: We recommend giving Docker at least 8 GB of RAM (On Docker Desktop you can go under settings -> resources) +***Note***: +1) We recommend giving Docker at least 8 GB of RAM (On Docker Desktop you can go under settings -> resources) +2) If you are a Windows user, we recommend you to install the *Windows Subsystem for Linux (WSL)* and integrate docker with WSL. WSL presence is highly recommended since many automation scripts written in Unix Shell (or `.sh`) format. For WSL installation's guidance can be found [here](https://docs.microsoft.com/en-us/windows/wsl/install). We use Ubuntu on WSL for debug and testing purposes. #### Demo correlating Uber traversals with Google popularities @@ -86,15 +88,14 @@ Linux/Mac: ```zsh cd kuwala/scripts && sh initialize_core_components.sh && sh run_cli.sh ``` -and for Windows (Please use PowerShell or any Docker integrated terminal): -```PS -cd kuwala/scripts && sh initialize_windows.sh && cd windows && sh initialize_core_components.sh && sh run_cli.sh -``` +Windows PowerShell with WSL: +1) Activate the WSL with `bash` command. +2) Run the same command as Linux/Mac above. #### Run the data pipelines yourself -To run the pipelines yourself, build the components first from inside the `kuwala/scripts` directory (or if the computer uses Windows, go to `kuwala/scripts/windows`) by executing the -`initialize_all_components.sh` script and the starting the CLI by running the `run_cli.sh` script. . +To run the pipelines yourself, build the components first from inside the `kuwala/scripts` directory (or if the computer uses Windows, activate WSL with `bash` command first) by executing the +`initialize_all_components.sh` script and the starting the CLI by running the `run_cli.sh` script. --- From 5126df134cd60da61c9f32a161ba338e15308053 Mon Sep 17 00:00:00 2001 From: IritaSee Date: Fri, 14 Jan 2022 22:51:41 +0700 Subject: [PATCH 05/15] convert all shell to python script --- kuwala/README.md | 5 +- kuwala/scripts/build_all_containers.py | 59 +++++++++++++++++ kuwala/scripts/build_all_containers.sh | 2 - kuwala/scripts/build_cli.py | 66 ++++++++++++++++++++ kuwala/scripts/build_cli.sh | 6 -- kuwala/scripts/build_jupyter_notebook.py | 57 +++++++++++++++++ kuwala/scripts/build_jupyter_notebook.sh | 2 - kuwala/scripts/build_neo4j.py | 56 +++++++++++++++++ kuwala/scripts/build_neo4j.sh | 2 - kuwala/scripts/create_zip_archive.py | 56 +++++++++++++++++ kuwala/scripts/create_zip_archive.sh | 2 - kuwala/scripts/initialize_all_components.py | 62 ++++++++++++++++++ kuwala/scripts/initialize_all_components.sh | 5 -- kuwala/scripts/initialize_core_components.py | 58 +++++++++++++++++ kuwala/scripts/initialize_core_components.sh | 3 - kuwala/scripts/initialize_git_submodules.py | 56 +++++++++++++++++ kuwala/scripts/initialize_git_submodules.sh | 2 - kuwala/scripts/run_cli.py | 60 ++++++++++++++++++ kuwala/scripts/run_cli.sh | 4 -- kuwala/scripts/run_jupyter_notebook.py | 57 +++++++++++++++++ kuwala/scripts/run_jupyter_notebook.sh | 2 - kuwala/scripts/stop_all_containers.py | 60 ++++++++++++++++++ kuwala/scripts/stop_all_containers.sh | 4 -- 23 files changed, 650 insertions(+), 36 deletions(-) create mode 100644 kuwala/scripts/build_all_containers.py delete mode 100644 kuwala/scripts/build_all_containers.sh create mode 100644 kuwala/scripts/build_cli.py delete mode 100644 kuwala/scripts/build_cli.sh create mode 100644 kuwala/scripts/build_jupyter_notebook.py delete mode 100644 kuwala/scripts/build_jupyter_notebook.sh create mode 100644 kuwala/scripts/build_neo4j.py delete mode 100644 kuwala/scripts/build_neo4j.sh create mode 100644 kuwala/scripts/create_zip_archive.py delete mode 100644 kuwala/scripts/create_zip_archive.sh create mode 100644 kuwala/scripts/initialize_all_components.py delete mode 100644 kuwala/scripts/initialize_all_components.sh create mode 100644 kuwala/scripts/initialize_core_components.py delete mode 100644 kuwala/scripts/initialize_core_components.sh create mode 100644 kuwala/scripts/initialize_git_submodules.py delete mode 100644 kuwala/scripts/initialize_git_submodules.sh create mode 100644 kuwala/scripts/run_cli.py delete mode 100644 kuwala/scripts/run_cli.sh create mode 100644 kuwala/scripts/run_jupyter_notebook.py delete mode 100644 kuwala/scripts/run_jupyter_notebook.sh create mode 100644 kuwala/scripts/stop_all_containers.py delete mode 100644 kuwala/scripts/stop_all_containers.sh diff --git a/kuwala/README.md b/kuwala/README.md index af734b28..2479889b 100644 --- a/kuwala/README.md +++ b/kuwala/README.md @@ -9,12 +9,13 @@ Installed version of *Docker* and *docker-compose* ### Pipelines -If you want to build all containers for all pipelines, change your working directory to `./kuwala/scripts` (or move to `./kuwala/scripts/`, run `initialize_windows.sh`, and change directory to `windows/` if you are running a Windows machine) and run: +If you want to build all containers for all pipelines, change your working directory to `./kuwala/scripts` and run: ```zsh -sh initialize_all_components.sh +python3 initialize_all_components.py ``` + You can also build the containers individually for single pipelines. All services are listed in the [`./docker-compose.yml`](https://github.com/kuwala-io/kuwala/tree/master/kuwala/docker-compose.yml). Please refer to each pipeline's `README.md` on how to run them. You can find the pipeline directories under diff --git a/kuwala/scripts/build_all_containers.py b/kuwala/scripts/build_all_containers.py new file mode 100644 index 00000000..c8be2547 --- /dev/null +++ b/kuwala/scripts/build_all_containers.py @@ -0,0 +1,59 @@ +""" +build_all_containers.sh: +cd .. +docker-compose build google-poi-api google-poi-pipeline neo4j-importer osm-parquetizer osm-poi population-density +""" + + +import subprocess +import os +from threading import Thread + +def run_command(command: [str], exit_keyword=None): + process = subprocess.Popen( + command, + bufsize=1, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + shell=True + ) + thread_result = dict(hit_exit_keyword=False) + + def print_std(std, result): + while True: + line = std.readline() + + if len(line.strip()) > 0: + print(line if 'Stage' not in line and '%' not in line else line.strip(), end='\r') + + if exit_keyword is not None and exit_keyword in line: + result['hit_exit_keyword'] = True + + break + + return_code = process.poll() + + if return_code is not None: + if return_code != 0: + return RuntimeError() + + break + + stdout_thread = Thread(target=print_std, args=(process.stdout, thread_result,), daemon=True) + stderr_thread = Thread(target=print_std, args=(process.stderr, thread_result,), daemon=True) + + stdout_thread.start() + stderr_thread.start() + + while stdout_thread.is_alive() and stderr_thread.is_alive(): + pass + + if thread_result['hit_exit_keyword']: + return process + +script_dir = os.path.dirname(os.path.abspath(__file__)) +os.chdir(os.path.join(script_dir,'..')) + +run_command(['docker-compose build google-poi-api google-poi-pipeline neo4j-importer osm-parquetizer osm-poi population-density']) + \ No newline at end of file diff --git a/kuwala/scripts/build_all_containers.sh b/kuwala/scripts/build_all_containers.sh deleted file mode 100644 index 41f20195..00000000 --- a/kuwala/scripts/build_all_containers.sh +++ /dev/null @@ -1,2 +0,0 @@ -cd .. -docker-compose build google-poi-api google-poi-pipeline neo4j-importer osm-parquetizer osm-poi population-density \ No newline at end of file diff --git a/kuwala/scripts/build_cli.py b/kuwala/scripts/build_cli.py new file mode 100644 index 00000000..80303c78 --- /dev/null +++ b/kuwala/scripts/build_cli.py @@ -0,0 +1,66 @@ +""" +build_cli.sh: + +cd ../.. +pip3 install virtualenv +virtualenv -p python3 venv +source ./venv/bin/activate +pip install -r kuwala/core/cli/requirements.txt +pip install -e . +""" + +import subprocess +import os +from threading import Thread + +def run_command(command: [str], exit_keyword=None): + process = subprocess.Popen( + command, + bufsize=1, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + shell=True + ) + thread_result = dict(hit_exit_keyword=False) + + def print_std(std, result): + while True: + line = std.readline() + + if len(line.strip()) > 0: + print(line if 'Stage' not in line and '%' not in line else line.strip(), end='\r') + + if exit_keyword is not None and exit_keyword in line: + result['hit_exit_keyword'] = True + + break + + return_code = process.poll() + + if return_code is not None: + if return_code != 0: + return RuntimeError() + + break + + stdout_thread = Thread(target=print_std, args=(process.stdout, thread_result,), daemon=True) + stderr_thread = Thread(target=print_std, args=(process.stderr, thread_result,), daemon=True) + + stdout_thread.start() + stderr_thread.start() + + while stdout_thread.is_alive() and stderr_thread.is_alive(): + pass + + if thread_result['hit_exit_keyword']: + return process + +script_dir = os.path.dirname(os.path.abspath(__file__)) + +os.chdir(os.path.join(script_dir,'../..')) +run_command(['pip3 install virtualenv']) +run_command(['virtualenv -p python3 venv']) +run_command(['source ./venv/bin/activate']) +run_command(['pip install -r kuwala/core/cli/requirements.txt']) +run_command(['pip install -e .']) \ No newline at end of file diff --git a/kuwala/scripts/build_cli.sh b/kuwala/scripts/build_cli.sh deleted file mode 100644 index ce6f1b70..00000000 --- a/kuwala/scripts/build_cli.sh +++ /dev/null @@ -1,6 +0,0 @@ -cd ../.. -pip3 install virtualenv -virtualenv -p python3 venv -source ./venv/bin/activate -pip install -r kuwala/core/cli/requirements.txt -pip install -e . \ No newline at end of file diff --git a/kuwala/scripts/build_jupyter_notebook.py b/kuwala/scripts/build_jupyter_notebook.py new file mode 100644 index 00000000..4870672d --- /dev/null +++ b/kuwala/scripts/build_jupyter_notebook.py @@ -0,0 +1,57 @@ +""" +build_jupyter_notebook.sh: +cd .. +docker-compose build jupyter + +""" +import subprocess +import os +from threading import Thread + +def run_command(command: [str], exit_keyword=None): + process = subprocess.Popen( + command, + bufsize=1, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + shell=True + ) + thread_result = dict(hit_exit_keyword=False) + + def print_std(std, result): + while True: + line = std.readline() + + if len(line.strip()) > 0: + print(line if 'Stage' not in line and '%' not in line else line.strip(), end='\r') + + if exit_keyword is not None and exit_keyword in line: + result['hit_exit_keyword'] = True + + break + + return_code = process.poll() + + if return_code is not None: + if return_code != 0: + return RuntimeError() + + break + + stdout_thread = Thread(target=print_std, args=(process.stdout, thread_result,), daemon=True) + stderr_thread = Thread(target=print_std, args=(process.stderr, thread_result,), daemon=True) + + stdout_thread.start() + stderr_thread.start() + + while stdout_thread.is_alive() and stderr_thread.is_alive(): + pass + + if thread_result['hit_exit_keyword']: + return process + +script_dir = os.path.dirname(os.path.abspath(__file__)) + +os.chdir(os.path.join(script_dir,'..')) +run_command(['docker-compose build jupyter']) \ No newline at end of file diff --git a/kuwala/scripts/build_jupyter_notebook.sh b/kuwala/scripts/build_jupyter_notebook.sh deleted file mode 100644 index b6320b2a..00000000 --- a/kuwala/scripts/build_jupyter_notebook.sh +++ /dev/null @@ -1,2 +0,0 @@ -cd .. -docker-compose build jupyter \ No newline at end of file diff --git a/kuwala/scripts/build_neo4j.py b/kuwala/scripts/build_neo4j.py new file mode 100644 index 00000000..f7f9ce6f --- /dev/null +++ b/kuwala/scripts/build_neo4j.py @@ -0,0 +1,56 @@ +""" +cd .. +docker-compose build neo4j +""" + +import subprocess +import os +from threading import Thread + +def run_command(command: [str], exit_keyword=None): + process = subprocess.Popen( + command, + bufsize=1, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + shell=True + ) + thread_result = dict(hit_exit_keyword=False) + + def print_std(std, result): + while True: + line = std.readline() + + if len(line.strip()) > 0: + print(line if 'Stage' not in line and '%' not in line else line.strip(), end='\r') + + if exit_keyword is not None and exit_keyword in line: + result['hit_exit_keyword'] = True + + break + + return_code = process.poll() + + if return_code is not None: + if return_code != 0: + return RuntimeError() + + break + + stdout_thread = Thread(target=print_std, args=(process.stdout, thread_result,), daemon=True) + stderr_thread = Thread(target=print_std, args=(process.stderr, thread_result,), daemon=True) + + stdout_thread.start() + stderr_thread.start() + + while stdout_thread.is_alive() and stderr_thread.is_alive(): + pass + + if thread_result['hit_exit_keyword']: + return process + +script_dir = os.path.dirname(os.path.abspath(__file__)) + +os.chdir(os.path.join(script_dir,'..')) +run_command(['docker-compose build neo4j']) \ No newline at end of file diff --git a/kuwala/scripts/build_neo4j.sh b/kuwala/scripts/build_neo4j.sh deleted file mode 100644 index 558a872b..00000000 --- a/kuwala/scripts/build_neo4j.sh +++ /dev/null @@ -1,2 +0,0 @@ -cd .. -docker-compose build neo4j \ No newline at end of file diff --git a/kuwala/scripts/create_zip_archive.py b/kuwala/scripts/create_zip_archive.py new file mode 100644 index 00000000..4938c5e2 --- /dev/null +++ b/kuwala/scripts/create_zip_archive.py @@ -0,0 +1,56 @@ +""" +cd ../.. +git archive --format=zip HEAD -o kuwala.zip +""" + +import subprocess +import os +from threading import Thread + +def run_command(command: [str], exit_keyword=None): + process = subprocess.Popen( + command, + bufsize=1, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + shell=True + ) + thread_result = dict(hit_exit_keyword=False) + + def print_std(std, result): + while True: + line = std.readline() + + if len(line.strip()) > 0: + print(line if 'Stage' not in line and '%' not in line else line.strip(), end='\r') + + if exit_keyword is not None and exit_keyword in line: + result['hit_exit_keyword'] = True + + break + + return_code = process.poll() + + if return_code is not None: + if return_code != 0: + return RuntimeError() + + break + + stdout_thread = Thread(target=print_std, args=(process.stdout, thread_result,), daemon=True) + stderr_thread = Thread(target=print_std, args=(process.stderr, thread_result,), daemon=True) + + stdout_thread.start() + stderr_thread.start() + + while stdout_thread.is_alive() and stderr_thread.is_alive(): + pass + + if thread_result['hit_exit_keyword']: + return process + +script_dir = os.path.dirname(os.path.abspath(__file__)) + +os.chdir(os.path.join(script_dir,'../..')) +run_command(['git archive --format=zip HEAD -o kuwala.zip']) \ No newline at end of file diff --git a/kuwala/scripts/create_zip_archive.sh b/kuwala/scripts/create_zip_archive.sh deleted file mode 100644 index a74b5584..00000000 --- a/kuwala/scripts/create_zip_archive.sh +++ /dev/null @@ -1,2 +0,0 @@ -cd ../.. -git archive --format=zip HEAD -o kuwala.zip \ No newline at end of file diff --git a/kuwala/scripts/initialize_all_components.py b/kuwala/scripts/initialize_all_components.py new file mode 100644 index 00000000..1a3ff644 --- /dev/null +++ b/kuwala/scripts/initialize_all_components.py @@ -0,0 +1,62 @@ +""" +sh initialize_git_submodules.sh +sh build_neo4j.sh +sh build_cli.sh +sh build_jupyter_notebook.sh +sh build_all_containers.sh +""" + +import subprocess +import os +from threading import Thread + +def run_command(command: [str], exit_keyword=None): + process = subprocess.Popen( + command, + bufsize=1, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + shell=True + ) + thread_result = dict(hit_exit_keyword=False) + + def print_std(std, result): + while True: + line = std.readline() + + if len(line.strip()) > 0: + print(line if 'Stage' not in line and '%' not in line else line.strip(), end='\r') + + if exit_keyword is not None and exit_keyword in line: + result['hit_exit_keyword'] = True + + break + + return_code = process.poll() + + if return_code is not None: + if return_code != 0: + return RuntimeError() + + break + + stdout_thread = Thread(target=print_std, args=(process.stdout, thread_result,), daemon=True) + stderr_thread = Thread(target=print_std, args=(process.stderr, thread_result,), daemon=True) + + stdout_thread.start() + stderr_thread.start() + + while stdout_thread.is_alive() and stderr_thread.is_alive(): + pass + + if thread_result['hit_exit_keyword']: + return process + +script_dir = os.path.dirname(os.path.abspath(__file__)) + +run_command(['python3 initialize_git_submodules.py']) +run_command(['python3 build_neo4j.py']) +run_command(['python3 build_cli.py']) +run_command(['python3 build_jupyter_notebook.py']) +run_command(['python3 build_all_containers.py']) diff --git a/kuwala/scripts/initialize_all_components.sh b/kuwala/scripts/initialize_all_components.sh deleted file mode 100644 index a6f73774..00000000 --- a/kuwala/scripts/initialize_all_components.sh +++ /dev/null @@ -1,5 +0,0 @@ -sh initialize_git_submodules.sh -sh build_neo4j.sh -sh build_cli.sh -sh build_jupyter_notebook.sh -sh build_all_containers.sh \ No newline at end of file diff --git a/kuwala/scripts/initialize_core_components.py b/kuwala/scripts/initialize_core_components.py new file mode 100644 index 00000000..c2acd170 --- /dev/null +++ b/kuwala/scripts/initialize_core_components.py @@ -0,0 +1,58 @@ +""" +sh build_neo4j.sh +sh build_cli.sh +sh build_jupyter_notebook.sh +""" + +import subprocess +import os +from threading import Thread + +def run_command(command: [str], exit_keyword=None): + process = subprocess.Popen( + command, + bufsize=1, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + shell=True + ) + thread_result = dict(hit_exit_keyword=False) + + def print_std(std, result): + while True: + line = std.readline() + + if len(line.strip()) > 0: + print(line if 'Stage' not in line and '%' not in line else line.strip(), end='\r') + + if exit_keyword is not None and exit_keyword in line: + result['hit_exit_keyword'] = True + + break + + return_code = process.poll() + + if return_code is not None: + if return_code != 0: + return RuntimeError() + + break + + stdout_thread = Thread(target=print_std, args=(process.stdout, thread_result,), daemon=True) + stderr_thread = Thread(target=print_std, args=(process.stderr, thread_result,), daemon=True) + + stdout_thread.start() + stderr_thread.start() + + while stdout_thread.is_alive() and stderr_thread.is_alive(): + pass + + if thread_result['hit_exit_keyword']: + return process + +script_dir = os.path.dirname(os.path.abspath(__file__)) + +run_command(['python3 build_neo4j.py']) +run_command(['python3 build_cli.py']) +run_command(['python3 build_jupyter_notebook.py']) diff --git a/kuwala/scripts/initialize_core_components.sh b/kuwala/scripts/initialize_core_components.sh deleted file mode 100644 index a5fdd4ec..00000000 --- a/kuwala/scripts/initialize_core_components.sh +++ /dev/null @@ -1,3 +0,0 @@ -sh build_neo4j.sh -sh build_cli.sh -sh build_jupyter_notebook.sh \ No newline at end of file diff --git a/kuwala/scripts/initialize_git_submodules.py b/kuwala/scripts/initialize_git_submodules.py new file mode 100644 index 00000000..95e887b6 --- /dev/null +++ b/kuwala/scripts/initialize_git_submodules.py @@ -0,0 +1,56 @@ +""" +cd ../.. +git submodule update --init --recursive +""" + +import subprocess +import os +from threading import Thread + +def run_command(command: [str], exit_keyword=None): + process = subprocess.Popen( + command, + bufsize=1, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + shell=True + ) + thread_result = dict(hit_exit_keyword=False) + + def print_std(std, result): + while True: + line = std.readline() + + if len(line.strip()) > 0: + print(line if 'Stage' not in line and '%' not in line else line.strip(), end='\r') + + if exit_keyword is not None and exit_keyword in line: + result['hit_exit_keyword'] = True + + break + + return_code = process.poll() + + if return_code is not None: + if return_code != 0: + return RuntimeError() + + break + + stdout_thread = Thread(target=print_std, args=(process.stdout, thread_result,), daemon=True) + stderr_thread = Thread(target=print_std, args=(process.stderr, thread_result,), daemon=True) + + stdout_thread.start() + stderr_thread.start() + + while stdout_thread.is_alive() and stderr_thread.is_alive(): + pass + + if thread_result['hit_exit_keyword']: + return process + +script_dir = os.path.dirname(os.path.abspath(__file__)) + +os.chdir(os.path.join(script_dir,'../..')) +run_command(['git submodule update --init --recursive']) \ No newline at end of file diff --git a/kuwala/scripts/initialize_git_submodules.sh b/kuwala/scripts/initialize_git_submodules.sh deleted file mode 100644 index 3e482e1e..00000000 --- a/kuwala/scripts/initialize_git_submodules.sh +++ /dev/null @@ -1,2 +0,0 @@ -cd ../.. -git submodule update --init --recursive \ No newline at end of file diff --git a/kuwala/scripts/run_cli.py b/kuwala/scripts/run_cli.py new file mode 100644 index 00000000..f4e2551c --- /dev/null +++ b/kuwala/scripts/run_cli.py @@ -0,0 +1,60 @@ +""" +cd ../../ +source ./venv/bin/activate +cd kuwala/core/cli +python3 src/main.py +""" + +import subprocess +import os +from threading import Thread + +def run_command(command: [str], exit_keyword=None): + process = subprocess.Popen( + command, + bufsize=1, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + shell=True + ) + thread_result = dict(hit_exit_keyword=False) + + def print_std(std, result): + while True: + line = std.readline() + + if len(line.strip()) > 0: + print(line if 'Stage' not in line and '%' not in line else line.strip(), end='\r') + + if exit_keyword is not None and exit_keyword in line: + result['hit_exit_keyword'] = True + + break + + return_code = process.poll() + + if return_code is not None: + if return_code != 0: + return RuntimeError() + + break + + stdout_thread = Thread(target=print_std, args=(process.stdout, thread_result,), daemon=True) + stderr_thread = Thread(target=print_std, args=(process.stderr, thread_result,), daemon=True) + + stdout_thread.start() + stderr_thread.start() + + while stdout_thread.is_alive() and stderr_thread.is_alive(): + pass + + if thread_result['hit_exit_keyword']: + return process + +script_dir = os.path.dirname(os.path.abspath(__file__)) + +os.chdir(os.path.join(script_dir,'../../')) +run_command(['source ./venv/bin/activate']) +os.chdir(os.path.join(script_dir,'../../','kuwala/core/cli')) +run_command(['python3 src/main.py']) diff --git a/kuwala/scripts/run_cli.sh b/kuwala/scripts/run_cli.sh deleted file mode 100644 index cbedab1d..00000000 --- a/kuwala/scripts/run_cli.sh +++ /dev/null @@ -1,4 +0,0 @@ -cd ../../ -source ./venv/bin/activate -cd kuwala/core/cli -python3 src/main.py \ No newline at end of file diff --git a/kuwala/scripts/run_jupyter_notebook.py b/kuwala/scripts/run_jupyter_notebook.py new file mode 100644 index 00000000..8f407278 --- /dev/null +++ b/kuwala/scripts/run_jupyter_notebook.py @@ -0,0 +1,57 @@ +""" +cd ../ +docker-compose run --service-ports jupyter +""" + +import subprocess +import os +from threading import Thread + +def run_command(command: [str], exit_keyword=None): + process = subprocess.Popen( + command, + bufsize=1, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + shell=True + ) + thread_result = dict(hit_exit_keyword=False) + + def print_std(std, result): + while True: + line = std.readline() + + if len(line.strip()) > 0: + print(line if 'Stage' not in line and '%' not in line else line.strip(), end='\r') + + if exit_keyword is not None and exit_keyword in line: + result['hit_exit_keyword'] = True + + break + + return_code = process.poll() + + if return_code is not None: + if return_code != 0: + return RuntimeError() + + break + + stdout_thread = Thread(target=print_std, args=(process.stdout, thread_result,), daemon=True) + stderr_thread = Thread(target=print_std, args=(process.stderr, thread_result,), daemon=True) + + stdout_thread.start() + stderr_thread.start() + + while stdout_thread.is_alive() and stderr_thread.is_alive(): + pass + + if thread_result['hit_exit_keyword']: + return process + +script_dir = os.path.dirname(os.path.abspath(__file__)) + +os.chdir(os.path.join(script_dir,'../')) +run_command(['docker-compose run --service-ports jupyter']) + diff --git a/kuwala/scripts/run_jupyter_notebook.sh b/kuwala/scripts/run_jupyter_notebook.sh deleted file mode 100644 index 432fb557..00000000 --- a/kuwala/scripts/run_jupyter_notebook.sh +++ /dev/null @@ -1,2 +0,0 @@ -cd ../ -docker-compose run --service-ports jupyter \ No newline at end of file diff --git a/kuwala/scripts/stop_all_containers.py b/kuwala/scripts/stop_all_containers.py new file mode 100644 index 00000000..effc4377 --- /dev/null +++ b/kuwala/scripts/stop_all_containers.py @@ -0,0 +1,60 @@ +""" +reset +docker stop $(docker ps -a -q) +docker-compose down +docker-compose rm -f +""" + +import subprocess +import os +from threading import Thread + +def run_command(command: [str], exit_keyword=None): + process = subprocess.Popen( + command, + bufsize=1, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + shell=True + ) + thread_result = dict(hit_exit_keyword=False) + + def print_std(std, result): + while True: + line = std.readline() + + if len(line.strip()) > 0: + print(line if 'Stage' not in line and '%' not in line else line.strip(), end='\r') + + if exit_keyword is not None and exit_keyword in line: + result['hit_exit_keyword'] = True + + break + + return_code = process.poll() + + if return_code is not None: + if return_code != 0: + return RuntimeError() + + break + + stdout_thread = Thread(target=print_std, args=(process.stdout, thread_result,), daemon=True) + stderr_thread = Thread(target=print_std, args=(process.stderr, thread_result,), daemon=True) + + stdout_thread.start() + stderr_thread.start() + + while stdout_thread.is_alive() and stderr_thread.is_alive(): + pass + + if thread_result['hit_exit_keyword']: + return process + +script_dir = os.path.dirname(os.path.abspath(__file__)) + +run_command(['reset']) +run_command(['docker stop $(docker ps -a -q)']) +run_command(['docker-compose down']) +run_command(['docker-compose rm -f']) \ No newline at end of file diff --git a/kuwala/scripts/stop_all_containers.sh b/kuwala/scripts/stop_all_containers.sh deleted file mode 100644 index be25dc8a..00000000 --- a/kuwala/scripts/stop_all_containers.sh +++ /dev/null @@ -1,4 +0,0 @@ -reset -docker stop $(docker ps -a -q) -docker-compose down -docker-compose rm -f \ No newline at end of file From 5e9c62d68730b3899ed7c43660232d28294b9275 Mon Sep 17 00:00:00 2001 From: IritaSee Date: Fri, 14 Jan 2022 22:58:00 +0700 Subject: [PATCH 06/15] readme adjustment --- README.md | 4 ++-- kuwala/README.md | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index cd730241..a553d2d2 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ Launch Docker in the background and from inside the root directory run: Linux/Mac: ```zsh -cd kuwala/scripts && sh initialize_core_components.sh && sh run_cli.sh +cd kuwala/scripts && python3 initialize_core_components.py && python3 run_cli.py ``` Windows PowerShell with WSL: 1) Activate the WSL with `bash` command. @@ -95,7 +95,7 @@ Windows PowerShell with WSL: #### Run the data pipelines yourself To run the pipelines yourself, build the components first from inside the `kuwala/scripts` directory (or if the computer uses Windows, activate WSL with `bash` command first) by executing the -`initialize_all_components.sh` script and the starting the CLI by running the `run_cli.sh` script. +`initialize_all_components.py` script and the starting the CLI by running the `run_cli.py` script. --- diff --git a/kuwala/README.md b/kuwala/README.md index 2479889b..21e50f86 100644 --- a/kuwala/README.md +++ b/kuwala/README.md @@ -34,20 +34,20 @@ Now you can proceed to any of the pipelines' `README.md` and follow the steps to ### Core -To initialize the CLI and Jupyter notebook run within the `./kuwala/scripts` directory (or `./kuwala/scripts/windows` ): +To initialize the CLI and Jupyter notebook run within the `./kuwala/scripts` directory: ```zsh -sh initialize_core_components.sh +python3 initialize_core_components.py ``` To launch the CLI run: ```zsh -sh run_cli.sh +python3 run_cli.py ``` If you only want to start the Jupyter environment run: ```zsh -sh run_jupyter_notebook.sh +python3 run_jupyter_notebook.py ``` \ No newline at end of file From b3b2f8f04c858495dc1ce52e2e22136c3b5f8c8a Mon Sep 17 00:00:00 2001 From: IritaSee Date: Fri, 14 Jan 2022 23:08:55 +0700 Subject: [PATCH 07/15] reworked run_command as module --- kuwala/scripts/build_all_containers.py | 49 +---------------- kuwala/scripts/build_cli.py | 56 +++----------------- kuwala/scripts/build_jupyter_notebook.py | 49 ++--------------- kuwala/scripts/build_neo4j.py | 48 +---------------- kuwala/scripts/create_zip_archive.py | 48 +---------------- kuwala/scripts/initialize_all_components.py | 56 +++----------------- kuwala/scripts/initialize_core_components.py | 52 ++---------------- kuwala/scripts/initialize_git_submodules.py | 48 +---------------- kuwala/scripts/run_cli.py | 50 ++--------------- kuwala/scripts/run_command.py | 51 ++++++++++++++++++ kuwala/scripts/run_jupyter_notebook.py | 49 +---------------- kuwala/scripts/stop_all_containers.py | 56 ++------------------ 12 files changed, 88 insertions(+), 524 deletions(-) create mode 100644 kuwala/scripts/run_command.py diff --git a/kuwala/scripts/build_all_containers.py b/kuwala/scripts/build_all_containers.py index c8be2547..24d8a475 100644 --- a/kuwala/scripts/build_all_containers.py +++ b/kuwala/scripts/build_all_containers.py @@ -4,56 +4,11 @@ docker-compose build google-poi-api google-poi-pipeline neo4j-importer osm-parquetizer osm-poi population-density """ - -import subprocess import os -from threading import Thread - -def run_command(command: [str], exit_keyword=None): - process = subprocess.Popen( - command, - bufsize=1, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - shell=True - ) - thread_result = dict(hit_exit_keyword=False) - - def print_std(std, result): - while True: - line = std.readline() - - if len(line.strip()) > 0: - print(line if 'Stage' not in line and '%' not in line else line.strip(), end='\r') - - if exit_keyword is not None and exit_keyword in line: - result['hit_exit_keyword'] = True - - break - - return_code = process.poll() - - if return_code is not None: - if return_code != 0: - return RuntimeError() - - break - - stdout_thread = Thread(target=print_std, args=(process.stdout, thread_result,), daemon=True) - stderr_thread = Thread(target=print_std, args=(process.stderr, thread_result,), daemon=True) - - stdout_thread.start() - stderr_thread.start() - - while stdout_thread.is_alive() and stderr_thread.is_alive(): - pass - - if thread_result['hit_exit_keyword']: - return process +import run_command as rc script_dir = os.path.dirname(os.path.abspath(__file__)) os.chdir(os.path.join(script_dir,'..')) -run_command(['docker-compose build google-poi-api google-poi-pipeline neo4j-importer osm-parquetizer osm-poi population-density']) +rc.run_command(['docker-compose build google-poi-api google-poi-pipeline neo4j-importer osm-parquetizer osm-poi population-density']) \ No newline at end of file diff --git a/kuwala/scripts/build_cli.py b/kuwala/scripts/build_cli.py index 80303c78..e13f45cc 100644 --- a/kuwala/scripts/build_cli.py +++ b/kuwala/scripts/build_cli.py @@ -9,58 +9,14 @@ pip install -e . """ -import subprocess import os -from threading import Thread - -def run_command(command: [str], exit_keyword=None): - process = subprocess.Popen( - command, - bufsize=1, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - shell=True - ) - thread_result = dict(hit_exit_keyword=False) - - def print_std(std, result): - while True: - line = std.readline() - - if len(line.strip()) > 0: - print(line if 'Stage' not in line and '%' not in line else line.strip(), end='\r') - - if exit_keyword is not None and exit_keyword in line: - result['hit_exit_keyword'] = True - - break - - return_code = process.poll() - - if return_code is not None: - if return_code != 0: - return RuntimeError() - - break - - stdout_thread = Thread(target=print_std, args=(process.stdout, thread_result,), daemon=True) - stderr_thread = Thread(target=print_std, args=(process.stderr, thread_result,), daemon=True) - - stdout_thread.start() - stderr_thread.start() - - while stdout_thread.is_alive() and stderr_thread.is_alive(): - pass - - if thread_result['hit_exit_keyword']: - return process +import run_command as rc script_dir = os.path.dirname(os.path.abspath(__file__)) os.chdir(os.path.join(script_dir,'../..')) -run_command(['pip3 install virtualenv']) -run_command(['virtualenv -p python3 venv']) -run_command(['source ./venv/bin/activate']) -run_command(['pip install -r kuwala/core/cli/requirements.txt']) -run_command(['pip install -e .']) \ No newline at end of file +rc.run_command(['pip3 install virtualenv']) +rc.run_command(['virtualenv -p python3 venv']) +rc.run_command(['source ./venv/bin/activate']) +rc.run_command(['pip install -r kuwala/core/cli/requirements.txt']) +rc.run_command(['pip install -e .']) \ No newline at end of file diff --git a/kuwala/scripts/build_jupyter_notebook.py b/kuwala/scripts/build_jupyter_notebook.py index 4870672d..1b75eb50 100644 --- a/kuwala/scripts/build_jupyter_notebook.py +++ b/kuwala/scripts/build_jupyter_notebook.py @@ -4,54 +4,11 @@ docker-compose build jupyter """ -import subprocess -import os -from threading import Thread - -def run_command(command: [str], exit_keyword=None): - process = subprocess.Popen( - command, - bufsize=1, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - shell=True - ) - thread_result = dict(hit_exit_keyword=False) - - def print_std(std, result): - while True: - line = std.readline() - - if len(line.strip()) > 0: - print(line if 'Stage' not in line and '%' not in line else line.strip(), end='\r') - - if exit_keyword is not None and exit_keyword in line: - result['hit_exit_keyword'] = True - - break - - return_code = process.poll() - if return_code is not None: - if return_code != 0: - return RuntimeError() - - break - - stdout_thread = Thread(target=print_std, args=(process.stdout, thread_result,), daemon=True) - stderr_thread = Thread(target=print_std, args=(process.stderr, thread_result,), daemon=True) - - stdout_thread.start() - stderr_thread.start() - - while stdout_thread.is_alive() and stderr_thread.is_alive(): - pass - - if thread_result['hit_exit_keyword']: - return process +import os +import run_command as rc script_dir = os.path.dirname(os.path.abspath(__file__)) os.chdir(os.path.join(script_dir,'..')) -run_command(['docker-compose build jupyter']) \ No newline at end of file +rc.run_command(['docker-compose build jupyter']) \ No newline at end of file diff --git a/kuwala/scripts/build_neo4j.py b/kuwala/scripts/build_neo4j.py index f7f9ce6f..ff488559 100644 --- a/kuwala/scripts/build_neo4j.py +++ b/kuwala/scripts/build_neo4j.py @@ -3,54 +3,10 @@ docker-compose build neo4j """ -import subprocess import os -from threading import Thread - -def run_command(command: [str], exit_keyword=None): - process = subprocess.Popen( - command, - bufsize=1, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - shell=True - ) - thread_result = dict(hit_exit_keyword=False) - - def print_std(std, result): - while True: - line = std.readline() - - if len(line.strip()) > 0: - print(line if 'Stage' not in line and '%' not in line else line.strip(), end='\r') - - if exit_keyword is not None and exit_keyword in line: - result['hit_exit_keyword'] = True - - break - - return_code = process.poll() - - if return_code is not None: - if return_code != 0: - return RuntimeError() - - break - - stdout_thread = Thread(target=print_std, args=(process.stdout, thread_result,), daemon=True) - stderr_thread = Thread(target=print_std, args=(process.stderr, thread_result,), daemon=True) - - stdout_thread.start() - stderr_thread.start() - - while stdout_thread.is_alive() and stderr_thread.is_alive(): - pass - - if thread_result['hit_exit_keyword']: - return process +import run_command as rc script_dir = os.path.dirname(os.path.abspath(__file__)) os.chdir(os.path.join(script_dir,'..')) -run_command(['docker-compose build neo4j']) \ No newline at end of file +rc.run_command(['docker-compose build neo4j']) \ No newline at end of file diff --git a/kuwala/scripts/create_zip_archive.py b/kuwala/scripts/create_zip_archive.py index 4938c5e2..fc6af1a1 100644 --- a/kuwala/scripts/create_zip_archive.py +++ b/kuwala/scripts/create_zip_archive.py @@ -3,54 +3,10 @@ git archive --format=zip HEAD -o kuwala.zip """ -import subprocess import os -from threading import Thread - -def run_command(command: [str], exit_keyword=None): - process = subprocess.Popen( - command, - bufsize=1, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - shell=True - ) - thread_result = dict(hit_exit_keyword=False) - - def print_std(std, result): - while True: - line = std.readline() - - if len(line.strip()) > 0: - print(line if 'Stage' not in line and '%' not in line else line.strip(), end='\r') - - if exit_keyword is not None and exit_keyword in line: - result['hit_exit_keyword'] = True - - break - - return_code = process.poll() - - if return_code is not None: - if return_code != 0: - return RuntimeError() - - break - - stdout_thread = Thread(target=print_std, args=(process.stdout, thread_result,), daemon=True) - stderr_thread = Thread(target=print_std, args=(process.stderr, thread_result,), daemon=True) - - stdout_thread.start() - stderr_thread.start() - - while stdout_thread.is_alive() and stderr_thread.is_alive(): - pass - - if thread_result['hit_exit_keyword']: - return process +import run_command as rc script_dir = os.path.dirname(os.path.abspath(__file__)) os.chdir(os.path.join(script_dir,'../..')) -run_command(['git archive --format=zip HEAD -o kuwala.zip']) \ No newline at end of file +rc.run_command(['git archive --format=zip HEAD -o kuwala.zip']) \ No newline at end of file diff --git a/kuwala/scripts/initialize_all_components.py b/kuwala/scripts/initialize_all_components.py index 1a3ff644..47bdbf46 100644 --- a/kuwala/scripts/initialize_all_components.py +++ b/kuwala/scripts/initialize_all_components.py @@ -6,57 +6,13 @@ sh build_all_containers.sh """ -import subprocess import os -from threading import Thread - -def run_command(command: [str], exit_keyword=None): - process = subprocess.Popen( - command, - bufsize=1, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - shell=True - ) - thread_result = dict(hit_exit_keyword=False) - - def print_std(std, result): - while True: - line = std.readline() - - if len(line.strip()) > 0: - print(line if 'Stage' not in line and '%' not in line else line.strip(), end='\r') - - if exit_keyword is not None and exit_keyword in line: - result['hit_exit_keyword'] = True - - break - - return_code = process.poll() - - if return_code is not None: - if return_code != 0: - return RuntimeError() - - break - - stdout_thread = Thread(target=print_std, args=(process.stdout, thread_result,), daemon=True) - stderr_thread = Thread(target=print_std, args=(process.stderr, thread_result,), daemon=True) - - stdout_thread.start() - stderr_thread.start() - - while stdout_thread.is_alive() and stderr_thread.is_alive(): - pass - - if thread_result['hit_exit_keyword']: - return process +import run_command as rc script_dir = os.path.dirname(os.path.abspath(__file__)) -run_command(['python3 initialize_git_submodules.py']) -run_command(['python3 build_neo4j.py']) -run_command(['python3 build_cli.py']) -run_command(['python3 build_jupyter_notebook.py']) -run_command(['python3 build_all_containers.py']) +rc.run_command(['python3 initialize_git_submodules.py']) +rc.run_command(['python3 build_neo4j.py']) +rc.run_command(['python3 build_cli.py']) +rc.run_command(['python3 build_jupyter_notebook.py']) +rc.run_command(['python3 build_all_containers.py']) diff --git a/kuwala/scripts/initialize_core_components.py b/kuwala/scripts/initialize_core_components.py index c2acd170..fe9e8e8c 100644 --- a/kuwala/scripts/initialize_core_components.py +++ b/kuwala/scripts/initialize_core_components.py @@ -4,55 +4,11 @@ sh build_jupyter_notebook.sh """ -import subprocess import os -from threading import Thread - -def run_command(command: [str], exit_keyword=None): - process = subprocess.Popen( - command, - bufsize=1, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - shell=True - ) - thread_result = dict(hit_exit_keyword=False) - - def print_std(std, result): - while True: - line = std.readline() - - if len(line.strip()) > 0: - print(line if 'Stage' not in line and '%' not in line else line.strip(), end='\r') - - if exit_keyword is not None and exit_keyword in line: - result['hit_exit_keyword'] = True - - break - - return_code = process.poll() - - if return_code is not None: - if return_code != 0: - return RuntimeError() - - break - - stdout_thread = Thread(target=print_std, args=(process.stdout, thread_result,), daemon=True) - stderr_thread = Thread(target=print_std, args=(process.stderr, thread_result,), daemon=True) - - stdout_thread.start() - stderr_thread.start() - - while stdout_thread.is_alive() and stderr_thread.is_alive(): - pass - - if thread_result['hit_exit_keyword']: - return process +import run_command as rc script_dir = os.path.dirname(os.path.abspath(__file__)) -run_command(['python3 build_neo4j.py']) -run_command(['python3 build_cli.py']) -run_command(['python3 build_jupyter_notebook.py']) +rc.run_command(['python3 build_neo4j.py']) +rc.run_command(['python3 build_cli.py']) +rc.run_command(['python3 build_jupyter_notebook.py']) diff --git a/kuwala/scripts/initialize_git_submodules.py b/kuwala/scripts/initialize_git_submodules.py index 95e887b6..2e00b096 100644 --- a/kuwala/scripts/initialize_git_submodules.py +++ b/kuwala/scripts/initialize_git_submodules.py @@ -3,54 +3,10 @@ git submodule update --init --recursive """ -import subprocess import os -from threading import Thread - -def run_command(command: [str], exit_keyword=None): - process = subprocess.Popen( - command, - bufsize=1, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - shell=True - ) - thread_result = dict(hit_exit_keyword=False) - - def print_std(std, result): - while True: - line = std.readline() - - if len(line.strip()) > 0: - print(line if 'Stage' not in line and '%' not in line else line.strip(), end='\r') - - if exit_keyword is not None and exit_keyword in line: - result['hit_exit_keyword'] = True - - break - - return_code = process.poll() - - if return_code is not None: - if return_code != 0: - return RuntimeError() - - break - - stdout_thread = Thread(target=print_std, args=(process.stdout, thread_result,), daemon=True) - stderr_thread = Thread(target=print_std, args=(process.stderr, thread_result,), daemon=True) - - stdout_thread.start() - stderr_thread.start() - - while stdout_thread.is_alive() and stderr_thread.is_alive(): - pass - - if thread_result['hit_exit_keyword']: - return process +import run_command as rc script_dir = os.path.dirname(os.path.abspath(__file__)) os.chdir(os.path.join(script_dir,'../..')) -run_command(['git submodule update --init --recursive']) \ No newline at end of file +rc.run_command(['git submodule update --init --recursive']) \ No newline at end of file diff --git a/kuwala/scripts/run_cli.py b/kuwala/scripts/run_cli.py index f4e2551c..3ea2aa03 100644 --- a/kuwala/scripts/run_cli.py +++ b/kuwala/scripts/run_cli.py @@ -5,56 +5,12 @@ python3 src/main.py """ -import subprocess import os -from threading import Thread - -def run_command(command: [str], exit_keyword=None): - process = subprocess.Popen( - command, - bufsize=1, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - shell=True - ) - thread_result = dict(hit_exit_keyword=False) - - def print_std(std, result): - while True: - line = std.readline() - - if len(line.strip()) > 0: - print(line if 'Stage' not in line and '%' not in line else line.strip(), end='\r') - - if exit_keyword is not None and exit_keyword in line: - result['hit_exit_keyword'] = True - - break - - return_code = process.poll() - - if return_code is not None: - if return_code != 0: - return RuntimeError() - - break - - stdout_thread = Thread(target=print_std, args=(process.stdout, thread_result,), daemon=True) - stderr_thread = Thread(target=print_std, args=(process.stderr, thread_result,), daemon=True) - - stdout_thread.start() - stderr_thread.start() - - while stdout_thread.is_alive() and stderr_thread.is_alive(): - pass - - if thread_result['hit_exit_keyword']: - return process +import run_command as rc script_dir = os.path.dirname(os.path.abspath(__file__)) os.chdir(os.path.join(script_dir,'../../')) -run_command(['source ./venv/bin/activate']) +rc.run_command(['source ./venv/bin/activate']) os.chdir(os.path.join(script_dir,'../../','kuwala/core/cli')) -run_command(['python3 src/main.py']) +rc.run_command(['python3 src/main.py']) diff --git a/kuwala/scripts/run_command.py b/kuwala/scripts/run_command.py new file mode 100644 index 00000000..eb9a568b --- /dev/null +++ b/kuwala/scripts/run_command.py @@ -0,0 +1,51 @@ +from threading import Thread +import subprocess +import os + +def run_command(command: [str], exit_keyword=None): + process = subprocess.Popen( + command, + bufsize=1, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + shell=True + ) + thread_result = dict(hit_exit_keyword=False) + + def print_std(std, result): + while True: + line = std.readline() + + if len(line.strip()) > 0: + print(line if 'Stage' not in line and '%' not in line else line.strip(), end='\r') + + if exit_keyword is not None and exit_keyword in line: + result['hit_exit_keyword'] = True + + break + + return_code = process.poll() + + if return_code is not None: + if return_code != 0: + return RuntimeError() + + break + + stdout_thread = Thread(target=print_std, args=(process.stdout, thread_result,), daemon=True) + stderr_thread = Thread(target=print_std, args=(process.stderr, thread_result,), daemon=True) + + stdout_thread.start() + stderr_thread.start() + + while stdout_thread.is_alive() and stderr_thread.is_alive(): + pass + + if thread_result['hit_exit_keyword']: + return process + + + +if __name__=='__main__': + run_command() \ No newline at end of file diff --git a/kuwala/scripts/run_jupyter_notebook.py b/kuwala/scripts/run_jupyter_notebook.py index 8f407278..39e9759f 100644 --- a/kuwala/scripts/run_jupyter_notebook.py +++ b/kuwala/scripts/run_jupyter_notebook.py @@ -2,56 +2,11 @@ cd ../ docker-compose run --service-ports jupyter """ - -import subprocess import os -from threading import Thread - -def run_command(command: [str], exit_keyword=None): - process = subprocess.Popen( - command, - bufsize=1, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - shell=True - ) - thread_result = dict(hit_exit_keyword=False) - - def print_std(std, result): - while True: - line = std.readline() - - if len(line.strip()) > 0: - print(line if 'Stage' not in line and '%' not in line else line.strip(), end='\r') - - if exit_keyword is not None and exit_keyword in line: - result['hit_exit_keyword'] = True - - break - - return_code = process.poll() - - if return_code is not None: - if return_code != 0: - return RuntimeError() - - break - - stdout_thread = Thread(target=print_std, args=(process.stdout, thread_result,), daemon=True) - stderr_thread = Thread(target=print_std, args=(process.stderr, thread_result,), daemon=True) - - stdout_thread.start() - stderr_thread.start() - - while stdout_thread.is_alive() and stderr_thread.is_alive(): - pass - - if thread_result['hit_exit_keyword']: - return process +import run_command as rc script_dir = os.path.dirname(os.path.abspath(__file__)) os.chdir(os.path.join(script_dir,'../')) -run_command(['docker-compose run --service-ports jupyter']) +rc.run_command(['docker-compose run --service-ports jupyter']) diff --git a/kuwala/scripts/stop_all_containers.py b/kuwala/scripts/stop_all_containers.py index effc4377..4cfccf1d 100644 --- a/kuwala/scripts/stop_all_containers.py +++ b/kuwala/scripts/stop_all_containers.py @@ -5,56 +5,10 @@ docker-compose rm -f """ -import subprocess import os -from threading import Thread +import run_command as rc -def run_command(command: [str], exit_keyword=None): - process = subprocess.Popen( - command, - bufsize=1, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - shell=True - ) - thread_result = dict(hit_exit_keyword=False) - - def print_std(std, result): - while True: - line = std.readline() - - if len(line.strip()) > 0: - print(line if 'Stage' not in line and '%' not in line else line.strip(), end='\r') - - if exit_keyword is not None and exit_keyword in line: - result['hit_exit_keyword'] = True - - break - - return_code = process.poll() - - if return_code is not None: - if return_code != 0: - return RuntimeError() - - break - - stdout_thread = Thread(target=print_std, args=(process.stdout, thread_result,), daemon=True) - stderr_thread = Thread(target=print_std, args=(process.stderr, thread_result,), daemon=True) - - stdout_thread.start() - stderr_thread.start() - - while stdout_thread.is_alive() and stderr_thread.is_alive(): - pass - - if thread_result['hit_exit_keyword']: - return process - -script_dir = os.path.dirname(os.path.abspath(__file__)) - -run_command(['reset']) -run_command(['docker stop $(docker ps -a -q)']) -run_command(['docker-compose down']) -run_command(['docker-compose rm -f']) \ No newline at end of file +rc.run_command(['reset']) +rc.run_command(['docker stop $(docker ps -a -q)']) +rc.run_command(['docker-compose down']) +rc.run_command(['docker-compose rm -f']) \ No newline at end of file From a3176f8b2bed6adeb1038f50c037d2c7ad4963c1 Mon Sep 17 00:00:00 2001 From: IritaSee Date: Wed, 19 Jan 2022 21:17:22 +0700 Subject: [PATCH 08/15] update osm readme --- kuwala/pipelines/osm-poi/README.md | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/kuwala/pipelines/osm-poi/README.md b/kuwala/pipelines/osm-poi/README.md index 9ce55165..2d821449 100644 --- a/kuwala/pipelines/osm-poi/README.md +++ b/kuwala/pipelines/osm-poi/README.md @@ -23,16 +23,8 @@ To transform the standard `pbf` files, which is the file format of the OSM data, OSM-parquetizer is based on a Git submodule which needs to be initialized first. To initialize the submodule, run from inside the root directory: -Linux/Mac: - -```zsh -cd kuwala/scripts && sh initialize_git_submodules.sh -``` - -Windows: - ```zsh -cd kuwala/scripts && sh initialize_windows.sh && cd windows && sh initialize_git_submodules.sh +cd kuwala/scripts && python3 initialize_git_submodules.py ``` To make sure you are running the latest version of the pipeline, build the Docker images from inside the `kuwala` From e0981a2c1c4c1faf9a4a1540ec4123778afb3a1b Mon Sep 17 00:00:00 2001 From: IritaSee Date: Wed, 19 Jan 2022 21:42:46 +0700 Subject: [PATCH 09/15] update readme, delete shell script --- kuwala/README.md | 2 +- kuwala/scripts/build_postgres.sh | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) delete mode 100644 kuwala/scripts/build_postgres.sh diff --git a/kuwala/README.md b/kuwala/README.md index 2f63788c..101631b4 100644 --- a/kuwala/README.md +++ b/kuwala/README.md @@ -25,7 +25,7 @@ each pipeline's `README.md` on how to run them. You can find the pipeline direct It would be safer to always run the build commands after pulling new code changes so that your local images have the latest code. -Docker images will only be built once when you run the `initialize_all_components.sh` script, so new changes in the code +Docker images will only be built once when you run the `initialize_all_components.py` script, so new changes in the code will not reflect on your local unless you explicitly run the build commands. Please note that the Docker runs will create data folders under `./kuwala/tmp/kuwala` that will be used for db, file diff --git a/kuwala/scripts/build_postgres.sh b/kuwala/scripts/build_postgres.sh deleted file mode 100644 index df71c8f5..00000000 --- a/kuwala/scripts/build_postgres.sh +++ /dev/null @@ -1,2 +0,0 @@ -cd .. -docker-compose build postgres \ No newline at end of file From a4aa0bac7d12bb1cab73ba58f217f064ca246273 Mon Sep 17 00:00:00 2001 From: IritaSee Date: Wed, 19 Jan 2022 21:48:33 +0700 Subject: [PATCH 10/15] main readme update --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c1ed9129..686f3a98 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ the next section. ***Note***: 1) We recommend giving Docker at least 8 GB of RAM (On Docker Desktop you can go under settings -> resources) -2) If you are a Windows user, we recommend you to install the *Windows Subsystem for Linux (WSL)* and integrate docker with WSL. WSL presence is highly recommended since many automation scripts written in Unix Shell (or `.sh`) format. For WSL installation's guidance can be found [here](https://docs.microsoft.com/en-us/windows/wsl/install). We use Ubuntu on WSL for debug and testing purposes. +2) If you are a Windows user, we recommend you to install the *Windows Subsystem for Linux (WSL)* and integrate docker with WSL. For WSL installation's guidance can be found [here](https://docs.microsoft.com/en-us/windows/wsl/install). We use Ubuntu on WSL for debug and testing purposes. #### Demo correlating Uber traversals with Google popularities From f3eef70d53d157e4ed9a2f98bb3fbd23088da6af Mon Sep 17 00:00:00 2001 From: IritaSee Date: Thu, 20 Jan 2022 17:09:40 +0700 Subject: [PATCH 11/15] update launch message --- kuwala/core/cli/src/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kuwala/core/cli/src/main.py b/kuwala/core/cli/src/main.py index 6bf9ddff..0242235c 100644 --- a/kuwala/core/cli/src/main.py +++ b/kuwala/core/cli/src/main.py @@ -10,7 +10,7 @@ def launch_jupyter_notebook(): run_command('docker-compose run --service-ports jupyter', exit_keyword='or http') webbrowser.open('http://localhost:8888/lab/tree/kuwala/notebooks/popularity_correlation.ipynb') - print('To stop and remove all Docker containers run the "sh stop_all_containers.sh"') + print('To stop and remove all Docker containers run the "python3 stop_all_containers.py"') if __name__ == '__main__': From 0ad964334637108e4869eb35714c6a166420b2e4 Mon Sep 17 00:00:00 2001 From: IritaSee Date: Tue, 25 Jan 2022 18:48:59 +0800 Subject: [PATCH 12/15] create shell scripts, rearrange python scripts --- kuwala/scripts/{ => python}/build_all_containers.py | 0 kuwala/scripts/{ => python}/build_cli.py | 0 kuwala/scripts/{ => python}/build_jupyter_notebook.py | 0 kuwala/scripts/{ => python}/build_postgres.py | 0 kuwala/scripts/{ => python}/create_zip_archive.py | 0 kuwala/scripts/{ => python}/initialize_all_components.py | 0 kuwala/scripts/{ => python}/initialize_core_components.py | 0 kuwala/scripts/{ => python}/initialize_git_submodules.py | 0 kuwala/scripts/{ => python}/run_cli.py | 0 kuwala/scripts/{ => python}/run_command.py | 0 kuwala/scripts/{ => python}/run_jupyter_notebook.py | 0 kuwala/scripts/{ => python}/stop_all_containers.py | 0 kuwala/scripts/shell/build_all_containers.sh | 3 +++ kuwala/scripts/shell/build_cli.sh | 7 +++++++ kuwala/scripts/shell/build_jupyter_notebook.sh | 3 +++ kuwala/scripts/shell/build_postgres.sh | 2 ++ kuwala/scripts/shell/create_zip_archive.sh | 3 +++ kuwala/scripts/shell/initialize_all_components.sh | 3 +++ kuwala/scripts/shell/initialize_core_components.sh | 3 +++ kuwala/scripts/shell/initialize_git_submodules.sh | 3 +++ kuwala/scripts/shell/run_cli.sh | 5 +++++ kuwala/scripts/shell/run_jupyter_notebook.sh | 3 +++ kuwala/scripts/shell/stop_all_containers.sh | 4 ++++ 23 files changed, 39 insertions(+) rename kuwala/scripts/{ => python}/build_all_containers.py (100%) rename kuwala/scripts/{ => python}/build_cli.py (100%) rename kuwala/scripts/{ => python}/build_jupyter_notebook.py (100%) rename kuwala/scripts/{ => python}/build_postgres.py (100%) rename kuwala/scripts/{ => python}/create_zip_archive.py (100%) rename kuwala/scripts/{ => python}/initialize_all_components.py (100%) rename kuwala/scripts/{ => python}/initialize_core_components.py (100%) rename kuwala/scripts/{ => python}/initialize_git_submodules.py (100%) rename kuwala/scripts/{ => python}/run_cli.py (100%) rename kuwala/scripts/{ => python}/run_command.py (100%) rename kuwala/scripts/{ => python}/run_jupyter_notebook.py (100%) rename kuwala/scripts/{ => python}/stop_all_containers.py (100%) create mode 100644 kuwala/scripts/shell/build_all_containers.sh create mode 100644 kuwala/scripts/shell/build_cli.sh create mode 100644 kuwala/scripts/shell/build_jupyter_notebook.sh create mode 100644 kuwala/scripts/shell/build_postgres.sh create mode 100644 kuwala/scripts/shell/create_zip_archive.sh create mode 100644 kuwala/scripts/shell/initialize_all_components.sh create mode 100644 kuwala/scripts/shell/initialize_core_components.sh create mode 100644 kuwala/scripts/shell/initialize_git_submodules.sh create mode 100644 kuwala/scripts/shell/run_cli.sh create mode 100644 kuwala/scripts/shell/run_jupyter_notebook.sh create mode 100644 kuwala/scripts/shell/stop_all_containers.sh diff --git a/kuwala/scripts/build_all_containers.py b/kuwala/scripts/python/build_all_containers.py similarity index 100% rename from kuwala/scripts/build_all_containers.py rename to kuwala/scripts/python/build_all_containers.py diff --git a/kuwala/scripts/build_cli.py b/kuwala/scripts/python/build_cli.py similarity index 100% rename from kuwala/scripts/build_cli.py rename to kuwala/scripts/python/build_cli.py diff --git a/kuwala/scripts/build_jupyter_notebook.py b/kuwala/scripts/python/build_jupyter_notebook.py similarity index 100% rename from kuwala/scripts/build_jupyter_notebook.py rename to kuwala/scripts/python/build_jupyter_notebook.py diff --git a/kuwala/scripts/build_postgres.py b/kuwala/scripts/python/build_postgres.py similarity index 100% rename from kuwala/scripts/build_postgres.py rename to kuwala/scripts/python/build_postgres.py diff --git a/kuwala/scripts/create_zip_archive.py b/kuwala/scripts/python/create_zip_archive.py similarity index 100% rename from kuwala/scripts/create_zip_archive.py rename to kuwala/scripts/python/create_zip_archive.py diff --git a/kuwala/scripts/initialize_all_components.py b/kuwala/scripts/python/initialize_all_components.py similarity index 100% rename from kuwala/scripts/initialize_all_components.py rename to kuwala/scripts/python/initialize_all_components.py diff --git a/kuwala/scripts/initialize_core_components.py b/kuwala/scripts/python/initialize_core_components.py similarity index 100% rename from kuwala/scripts/initialize_core_components.py rename to kuwala/scripts/python/initialize_core_components.py diff --git a/kuwala/scripts/initialize_git_submodules.py b/kuwala/scripts/python/initialize_git_submodules.py similarity index 100% rename from kuwala/scripts/initialize_git_submodules.py rename to kuwala/scripts/python/initialize_git_submodules.py diff --git a/kuwala/scripts/run_cli.py b/kuwala/scripts/python/run_cli.py similarity index 100% rename from kuwala/scripts/run_cli.py rename to kuwala/scripts/python/run_cli.py diff --git a/kuwala/scripts/run_command.py b/kuwala/scripts/python/run_command.py similarity index 100% rename from kuwala/scripts/run_command.py rename to kuwala/scripts/python/run_command.py diff --git a/kuwala/scripts/run_jupyter_notebook.py b/kuwala/scripts/python/run_jupyter_notebook.py similarity index 100% rename from kuwala/scripts/run_jupyter_notebook.py rename to kuwala/scripts/python/run_jupyter_notebook.py diff --git a/kuwala/scripts/stop_all_containers.py b/kuwala/scripts/python/stop_all_containers.py similarity index 100% rename from kuwala/scripts/stop_all_containers.py rename to kuwala/scripts/python/stop_all_containers.py diff --git a/kuwala/scripts/shell/build_all_containers.sh b/kuwala/scripts/shell/build_all_containers.sh new file mode 100644 index 00000000..3e192d20 --- /dev/null +++ b/kuwala/scripts/shell/build_all_containers.sh @@ -0,0 +1,3 @@ +cd .. +cd .. +docker-compose build postgres database-importer database-transformer jupyter admin-boundaries google-poi-api google-poi-pipeline google-trends osm-parquetizer osm-poi population-density \ No newline at end of file diff --git a/kuwala/scripts/shell/build_cli.sh b/kuwala/scripts/shell/build_cli.sh new file mode 100644 index 00000000..7eeb839c --- /dev/null +++ b/kuwala/scripts/shell/build_cli.sh @@ -0,0 +1,7 @@ +cd ../.. +cd .. +pip3 install virtualenv +virtualenv -p python3 venv +source ./venv/bin/activate +pip install -r kuwala/core/cli/requirements.txt +pip install -e . \ No newline at end of file diff --git a/kuwala/scripts/shell/build_jupyter_notebook.sh b/kuwala/scripts/shell/build_jupyter_notebook.sh new file mode 100644 index 00000000..a582e610 --- /dev/null +++ b/kuwala/scripts/shell/build_jupyter_notebook.sh @@ -0,0 +1,3 @@ +cd .. +cd .. +docker-compose build jupyter \ No newline at end of file diff --git a/kuwala/scripts/shell/build_postgres.sh b/kuwala/scripts/shell/build_postgres.sh new file mode 100644 index 00000000..df71c8f5 --- /dev/null +++ b/kuwala/scripts/shell/build_postgres.sh @@ -0,0 +1,2 @@ +cd .. +docker-compose build postgres \ No newline at end of file diff --git a/kuwala/scripts/shell/create_zip_archive.sh b/kuwala/scripts/shell/create_zip_archive.sh new file mode 100644 index 00000000..57cfd984 --- /dev/null +++ b/kuwala/scripts/shell/create_zip_archive.sh @@ -0,0 +1,3 @@ +cd ../.. +cd .. +git archive --format=zip HEAD -o kuwala.zip \ No newline at end of file diff --git a/kuwala/scripts/shell/initialize_all_components.sh b/kuwala/scripts/shell/initialize_all_components.sh new file mode 100644 index 00000000..08834bc3 --- /dev/null +++ b/kuwala/scripts/shell/initialize_all_components.sh @@ -0,0 +1,3 @@ +sh initialize_git_submodules.sh +sh build_cli.sh +sh build_all_containers.sh \ No newline at end of file diff --git a/kuwala/scripts/shell/initialize_core_components.sh b/kuwala/scripts/shell/initialize_core_components.sh new file mode 100644 index 00000000..90cd4cca --- /dev/null +++ b/kuwala/scripts/shell/initialize_core_components.sh @@ -0,0 +1,3 @@ +sh build_postgres.sh +sh build_cli.sh +sh build_jupyter_notebook.sh \ No newline at end of file diff --git a/kuwala/scripts/shell/initialize_git_submodules.sh b/kuwala/scripts/shell/initialize_git_submodules.sh new file mode 100644 index 00000000..bc3b7dc4 --- /dev/null +++ b/kuwala/scripts/shell/initialize_git_submodules.sh @@ -0,0 +1,3 @@ +cd ../.. +cd .. +git submodule update --init --recursive \ No newline at end of file diff --git a/kuwala/scripts/shell/run_cli.sh b/kuwala/scripts/shell/run_cli.sh new file mode 100644 index 00000000..311d515a --- /dev/null +++ b/kuwala/scripts/shell/run_cli.sh @@ -0,0 +1,5 @@ +cd ../../ +cd .. +source ./venv/bin/activate +cd kuwala/core/cli +python3 src/main.py \ No newline at end of file diff --git a/kuwala/scripts/shell/run_jupyter_notebook.sh b/kuwala/scripts/shell/run_jupyter_notebook.sh new file mode 100644 index 00000000..cf08babb --- /dev/null +++ b/kuwala/scripts/shell/run_jupyter_notebook.sh @@ -0,0 +1,3 @@ +cd ../ +cd .. +docker-compose run --service-ports jupyter \ No newline at end of file diff --git a/kuwala/scripts/shell/stop_all_containers.sh b/kuwala/scripts/shell/stop_all_containers.sh new file mode 100644 index 00000000..be25dc8a --- /dev/null +++ b/kuwala/scripts/shell/stop_all_containers.sh @@ -0,0 +1,4 @@ +reset +docker stop $(docker ps -a -q) +docker-compose down +docker-compose rm -f \ No newline at end of file From b66dce3f82421023e25e1c7dc019bee7955464f2 Mon Sep 17 00:00:00 2001 From: IritaSee Date: Tue, 25 Jan 2022 19:31:38 +0800 Subject: [PATCH 13/15] fix main readme, fix directory changes in /python --- README.md | 11 ++++++----- kuwala/scripts/python/build_all_containers.py | 2 +- kuwala/scripts/python/build_cli.py | 2 +- kuwala/scripts/python/build_jupyter_notebook.py | 2 +- kuwala/scripts/python/build_postgres.py | 2 +- kuwala/scripts/python/create_zip_archive.py | 2 +- kuwala/scripts/python/initialize_git_submodules.py | 2 +- kuwala/scripts/python/run_cli.py | 4 ++-- kuwala/scripts/python/run_jupyter_notebook.py | 2 +- 9 files changed, 15 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 686f3a98..e1c839d6 100644 --- a/README.md +++ b/README.md @@ -89,14 +89,15 @@ You could either use the deployed example on Binder using the badge above or run simply uses Pandas dataframes and is not connecting to a data warehouse. \ To run the demo locally, launch Docker in the background and from inside the root directory run: -Linux/Mac: ```zsh -cd kuwala/scripts && python3 initialize_core_components.py && python3 run_cli.py +cd kuwala/scripts/shell && sh initialize_core_components.sh && sh run_cli.sh ``` -Windows PowerShell with WSL: -1) Activate the WSL with `bash` command. -2) Run the same command as Linux/Mac above. +Or if you are a Windows User having issue to run a `shell` script, you can run a `python` script as an alternative: +```zsh +cd kuwala/scripts/python && python3 initialize_core_components.py && python3 run_cli.py +``` +All `shell` scripts are also available in `python` script using the same file naming, inside the `/python` directory #### Run the data pipelines yourself To run the pipelines yourself, please follow the instructions for the diff --git a/kuwala/scripts/python/build_all_containers.py b/kuwala/scripts/python/build_all_containers.py index f23662f4..c58bdf53 100644 --- a/kuwala/scripts/python/build_all_containers.py +++ b/kuwala/scripts/python/build_all_containers.py @@ -8,7 +8,7 @@ import run_command as rc script_dir = os.path.dirname(os.path.abspath(__file__)) -os.chdir(os.path.join(script_dir,'..')) +os.chdir(os.path.join(script_dir,'../../')) rc.run_command(['docker-compose build postgres database-importer database-transformer jupyter admin-boundaries google-poi-api google-poi-pipeline google-trends osm-parquetizer osm-poi population-density']) \ No newline at end of file diff --git a/kuwala/scripts/python/build_cli.py b/kuwala/scripts/python/build_cli.py index e13f45cc..63a16b86 100644 --- a/kuwala/scripts/python/build_cli.py +++ b/kuwala/scripts/python/build_cli.py @@ -14,7 +14,7 @@ script_dir = os.path.dirname(os.path.abspath(__file__)) -os.chdir(os.path.join(script_dir,'../..')) +os.chdir(os.path.join(script_dir,'../../../')) rc.run_command(['pip3 install virtualenv']) rc.run_command(['virtualenv -p python3 venv']) rc.run_command(['source ./venv/bin/activate']) diff --git a/kuwala/scripts/python/build_jupyter_notebook.py b/kuwala/scripts/python/build_jupyter_notebook.py index 1b75eb50..14f94bf1 100644 --- a/kuwala/scripts/python/build_jupyter_notebook.py +++ b/kuwala/scripts/python/build_jupyter_notebook.py @@ -10,5 +10,5 @@ script_dir = os.path.dirname(os.path.abspath(__file__)) -os.chdir(os.path.join(script_dir,'..')) +os.chdir(os.path.join(script_dir,'../../')) rc.run_command(['docker-compose build jupyter']) \ No newline at end of file diff --git a/kuwala/scripts/python/build_postgres.py b/kuwala/scripts/python/build_postgres.py index ea7c5187..fdc98bd0 100644 --- a/kuwala/scripts/python/build_postgres.py +++ b/kuwala/scripts/python/build_postgres.py @@ -8,5 +8,5 @@ script_dir = os.path.dirname(os.path.abspath(__file__)) -os.chdir(os.path.join(script_dir,'..')) +os.chdir(os.path.join(script_dir,'../../')) rc.run_command(['docker-compose build postgres']) \ No newline at end of file diff --git a/kuwala/scripts/python/create_zip_archive.py b/kuwala/scripts/python/create_zip_archive.py index fc6af1a1..c3b5d26b 100644 --- a/kuwala/scripts/python/create_zip_archive.py +++ b/kuwala/scripts/python/create_zip_archive.py @@ -8,5 +8,5 @@ script_dir = os.path.dirname(os.path.abspath(__file__)) -os.chdir(os.path.join(script_dir,'../..')) +os.chdir(os.path.join(script_dir,'../../../')) rc.run_command(['git archive --format=zip HEAD -o kuwala.zip']) \ No newline at end of file diff --git a/kuwala/scripts/python/initialize_git_submodules.py b/kuwala/scripts/python/initialize_git_submodules.py index 2e00b096..37e94d6b 100644 --- a/kuwala/scripts/python/initialize_git_submodules.py +++ b/kuwala/scripts/python/initialize_git_submodules.py @@ -8,5 +8,5 @@ script_dir = os.path.dirname(os.path.abspath(__file__)) -os.chdir(os.path.join(script_dir,'../..')) +os.chdir(os.path.join(script_dir,'../../../')) rc.run_command(['git submodule update --init --recursive']) \ No newline at end of file diff --git a/kuwala/scripts/python/run_cli.py b/kuwala/scripts/python/run_cli.py index 3ea2aa03..ed2c26f7 100644 --- a/kuwala/scripts/python/run_cli.py +++ b/kuwala/scripts/python/run_cli.py @@ -10,7 +10,7 @@ script_dir = os.path.dirname(os.path.abspath(__file__)) -os.chdir(os.path.join(script_dir,'../../')) +os.chdir(os.path.join(script_dir,'../../../')) rc.run_command(['source ./venv/bin/activate']) -os.chdir(os.path.join(script_dir,'../../','kuwala/core/cli')) +os.chdir(os.path.join(script_dir,'../../../','kuwala/core/cli')) rc.run_command(['python3 src/main.py']) diff --git a/kuwala/scripts/python/run_jupyter_notebook.py b/kuwala/scripts/python/run_jupyter_notebook.py index 39e9759f..cd10d616 100644 --- a/kuwala/scripts/python/run_jupyter_notebook.py +++ b/kuwala/scripts/python/run_jupyter_notebook.py @@ -7,6 +7,6 @@ script_dir = os.path.dirname(os.path.abspath(__file__)) -os.chdir(os.path.join(script_dir,'../')) +os.chdir(os.path.join(script_dir,'../../')) rc.run_command(['docker-compose run --service-ports jupyter']) From aaea08a77fa100f57ad76244da902d07f4ae9007 Mon Sep 17 00:00:00 2001 From: IritaSee Date: Tue, 25 Jan 2022 19:47:36 +0800 Subject: [PATCH 14/15] update readme to follow dir changes --- README.md | 2 +- kuwala/core/cli/README.md | 21 +++++++++++++-------- kuwala/pipelines/osm-poi/README.md | 9 ++++++++- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index e1c839d6..c51831d3 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ Or if you are a Windows User having issue to run a `shell` script, you can run a ```zsh cd kuwala/scripts/python && python3 initialize_core_components.py && python3 run_cli.py ``` -All `shell` scripts are also available in `python` script using the same file naming, inside the `/python` directory +*All `shell` scripts are also available in `python` script using the same file naming, inside the `/python` directory* #### Run the data pipelines yourself To run the pipelines yourself, please follow the instructions for the diff --git a/kuwala/core/cli/README.md b/kuwala/core/cli/README.md index ec9dfaa8..c9de6271 100644 --- a/kuwala/core/cli/README.md +++ b/kuwala/core/cli/README.md @@ -20,18 +20,23 @@ The following pipelines can currently be selected through the CLI: To make sure you are running the latest version of all pipelines, run from inside the root directory: -Linux/Mac: ```zsh -cd kuwala/scripts && python3 initialize_all_components.py +cd kuwala/scripts/shell && sh initialize_all_components.sh ``` -Windows PowerShell with WSL: -1) Activate the WSL with `bash` command. -2) Run the same command as Linux/Mac above. +Or if you are a Windows User having issue to run a `shell` script, you can run a `python` script as an alternative: -To start the CLI, run the following script from inside the `kuwala/scripts` directory and follow the instructions: +```zsh +cd kuwala/scripts/python && python3 initialize_all_components.py +``` + +*All `shell` scripts are also available in `python` script using the same file naming, inside the `/python` directory* + + +To start the CLI, run the following script from inside the `kuwala/scripts/shell` directory and follow the instructions: ```zsh -python3 run_cli.py -``` \ No newline at end of file +sh run_cli.sh +``` +or `python3 run_cli.py` from inside the `kuwala/scripts/python` directory \ No newline at end of file diff --git a/kuwala/pipelines/osm-poi/README.md b/kuwala/pipelines/osm-poi/README.md index 2d821449..c0edb943 100644 --- a/kuwala/pipelines/osm-poi/README.md +++ b/kuwala/pipelines/osm-poi/README.md @@ -24,8 +24,15 @@ OSM-parquetizer is based on a Git submodule which needs to be initialized first. inside the root directory: ```zsh -cd kuwala/scripts && python3 initialize_git_submodules.py +cd kuwala/scripts/shell && sh initialize_git_submodules.sh ``` +Or if you are a Windows User having issue to run a `shell` script, you can run a `python` script as an alternative: + +```zsh +cd kuwala/scripts/python && python3 initialize_git_submodules.py +``` + +*All `shell` scripts are also available in `python` script using the same file naming, inside the `/python` directory* To make sure you are running the latest version of the pipeline, build the Docker images from inside the `kuwala` directory by running: From 5ac2c59cc6a8a2b4fa1d11485f8fbdf2c1e36c51 Mon Sep 17 00:00:00 2001 From: IritaSee Date: Thu, 27 Jan 2022 12:44:59 +0800 Subject: [PATCH 15/15] fix spelling --- README.md | 2 +- kuwala/README.md | 19 +++++++++++-------- kuwala/core/cli/README.md | 2 +- kuwala/core/cli/src/main.py | 2 +- kuwala/pipelines/osm-poi/README.md | 2 +- kuwala/scripts/shell/build_all_containers.sh | 3 +-- kuwala/scripts/shell/build_cli.sh | 3 +-- .../scripts/shell/build_jupyter_notebook.sh | 3 +-- kuwala/scripts/shell/create_zip_archive.sh | 3 +-- .../shell/initialize_git_submodules.sh | 3 +-- kuwala/scripts/shell/run_cli.sh | 3 +-- kuwala/scripts/shell/run_jupyter_notebook.sh | 3 +-- 12 files changed, 22 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index c51831d3..1ef7f10a 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ To run the demo locally, launch Docker in the background and from inside the roo ```zsh cd kuwala/scripts/shell && sh initialize_core_components.sh && sh run_cli.sh ``` -Or if you are a Windows User having issue to run a `shell` script, you can run a `python` script as an alternative: +Or if you are a Windows user having issues to run a `shell` script, you can run a `python` script as an alternative: ```zsh cd kuwala/scripts/python && python3 initialize_core_components.py && python3 run_cli.py diff --git a/kuwala/README.md b/kuwala/README.md index 101631b4..2b5c4d43 100644 --- a/kuwala/README.md +++ b/kuwala/README.md @@ -9,11 +9,11 @@ Installed version of *Docker* and *docker-compose v2* ### Pipelines -If you want to build all containers for all pipelines, change your working directory to `./kuwala/scripts` and run: +If you want to build all containers for all pipelines, change your working directory to `./kuwala/scripts/shell` and run: ```zsh -python3 initialize_all_components.py +sh initialize_all_components.sh ``` @@ -25,7 +25,7 @@ each pipeline's `README.md` on how to run them. You can find the pipeline direct It would be safer to always run the build commands after pulling new code changes so that your local images have the latest code. -Docker images will only be built once when you run the `initialize_all_components.py` script, so new changes in the code +Docker images will only be built once when you run the `initialize_all_components.sh` script, so new changes in the code will not reflect on your local unless you explicitly run the build commands. Please note that the Docker runs will create data folders under `./kuwala/tmp/kuwala` that will be used for db, file @@ -35,20 +35,23 @@ Now you can proceed to any of the pipelines' `README.md` and follow the steps to ### Core -To initialize the CLI and Jupyter notebook run within the `./kuwala/scripts` directory: +To initialize the CLI and Jupyter notebook run within the `./kuwala/scripts/shell` directory: ```zsh -python3 initialize_core_components.py +sh initialize_core_components.sh ``` To launch the CLI run: ```zsh -python3 run_cli.py +sh run_cli.sh ``` If you only want to start the Jupyter environment run: ```zsh -python3 run_jupyter_notebook.py -``` \ No newline at end of file +sh run_jupyter_notebook.sh +``` +Or if you are a Windows user having issues to run a `shell` script, you can run a `python` script as an alternative. For example you can run `initialize_core_components.py` under `kuwala/scripts/python`.\ +\ +*All `shell` scripts are also available in `python` script using the same file naming, inside the `kuwala/scripts/python` directory* \ No newline at end of file diff --git a/kuwala/core/cli/README.md b/kuwala/core/cli/README.md index c9de6271..f85d823e 100644 --- a/kuwala/core/cli/README.md +++ b/kuwala/core/cli/README.md @@ -25,7 +25,7 @@ To make sure you are running the latest version of all pipelines, run from insid cd kuwala/scripts/shell && sh initialize_all_components.sh ``` -Or if you are a Windows User having issue to run a `shell` script, you can run a `python` script as an alternative: +Or if you are a Windows user having issues to run a `shell` script, you can run a `python` script as an alternative: ```zsh cd kuwala/scripts/python && python3 initialize_all_components.py diff --git a/kuwala/core/cli/src/main.py b/kuwala/core/cli/src/main.py index 0242235c..a41f97d6 100644 --- a/kuwala/core/cli/src/main.py +++ b/kuwala/core/cli/src/main.py @@ -10,7 +10,7 @@ def launch_jupyter_notebook(): run_command('docker-compose run --service-ports jupyter', exit_keyword='or http') webbrowser.open('http://localhost:8888/lab/tree/kuwala/notebooks/popularity_correlation.ipynb') - print('To stop and remove all Docker containers run the "python3 stop_all_containers.py"') + print('To stop and remove all Docker containers run the "sh stop_all_containers.sh" or "python3 stop_all_containers.py"') if __name__ == '__main__': diff --git a/kuwala/pipelines/osm-poi/README.md b/kuwala/pipelines/osm-poi/README.md index c0edb943..80dfe567 100644 --- a/kuwala/pipelines/osm-poi/README.md +++ b/kuwala/pipelines/osm-poi/README.md @@ -26,7 +26,7 @@ inside the root directory: ```zsh cd kuwala/scripts/shell && sh initialize_git_submodules.sh ``` -Or if you are a Windows User having issue to run a `shell` script, you can run a `python` script as an alternative: +Or if you are a Windows user having issues to run a a `shell` script, you can run a `python` script as an alternative: ```zsh cd kuwala/scripts/python && python3 initialize_git_submodules.py diff --git a/kuwala/scripts/shell/build_all_containers.sh b/kuwala/scripts/shell/build_all_containers.sh index 3e192d20..3997f87b 100644 --- a/kuwala/scripts/shell/build_all_containers.sh +++ b/kuwala/scripts/shell/build_all_containers.sh @@ -1,3 +1,2 @@ -cd .. -cd .. +cd ../.. docker-compose build postgres database-importer database-transformer jupyter admin-boundaries google-poi-api google-poi-pipeline google-trends osm-parquetizer osm-poi population-density \ No newline at end of file diff --git a/kuwala/scripts/shell/build_cli.sh b/kuwala/scripts/shell/build_cli.sh index 7eeb839c..c5299afc 100644 --- a/kuwala/scripts/shell/build_cli.sh +++ b/kuwala/scripts/shell/build_cli.sh @@ -1,5 +1,4 @@ -cd ../.. -cd .. +cd ../../.. pip3 install virtualenv virtualenv -p python3 venv source ./venv/bin/activate diff --git a/kuwala/scripts/shell/build_jupyter_notebook.sh b/kuwala/scripts/shell/build_jupyter_notebook.sh index a582e610..bf29848f 100644 --- a/kuwala/scripts/shell/build_jupyter_notebook.sh +++ b/kuwala/scripts/shell/build_jupyter_notebook.sh @@ -1,3 +1,2 @@ -cd .. -cd .. +cd ../.. docker-compose build jupyter \ No newline at end of file diff --git a/kuwala/scripts/shell/create_zip_archive.sh b/kuwala/scripts/shell/create_zip_archive.sh index 57cfd984..23124d24 100644 --- a/kuwala/scripts/shell/create_zip_archive.sh +++ b/kuwala/scripts/shell/create_zip_archive.sh @@ -1,3 +1,2 @@ -cd ../.. -cd .. +cd ../../.. git archive --format=zip HEAD -o kuwala.zip \ No newline at end of file diff --git a/kuwala/scripts/shell/initialize_git_submodules.sh b/kuwala/scripts/shell/initialize_git_submodules.sh index bc3b7dc4..da0e44e2 100644 --- a/kuwala/scripts/shell/initialize_git_submodules.sh +++ b/kuwala/scripts/shell/initialize_git_submodules.sh @@ -1,3 +1,2 @@ -cd ../.. -cd .. +cd ../../.. git submodule update --init --recursive \ No newline at end of file diff --git a/kuwala/scripts/shell/run_cli.sh b/kuwala/scripts/shell/run_cli.sh index 311d515a..7e370716 100644 --- a/kuwala/scripts/shell/run_cli.sh +++ b/kuwala/scripts/shell/run_cli.sh @@ -1,5 +1,4 @@ -cd ../../ -cd .. +cd ../../../ source ./venv/bin/activate cd kuwala/core/cli python3 src/main.py \ No newline at end of file diff --git a/kuwala/scripts/shell/run_jupyter_notebook.sh b/kuwala/scripts/shell/run_jupyter_notebook.sh index cf08babb..6a482838 100644 --- a/kuwala/scripts/shell/run_jupyter_notebook.sh +++ b/kuwala/scripts/shell/run_jupyter_notebook.sh @@ -1,3 +1,2 @@ -cd ../ -cd .. +cd ../.. docker-compose run --service-ports jupyter \ No newline at end of file