From d1e9b6b5a3fd42307220d15e99f4b3a985e60acb Mon Sep 17 00:00:00 2001 From: Jonathan de Bruin Date: Mon, 6 Nov 2023 15:18:45 +0100 Subject: [PATCH] Refactor development workflow for flask app (#1568) --- DEVELOPMENT.md | 235 ++++---- Docker/auth_verified/wsgi.py | 2 +- asreview/entry_points/lab.py | 249 +++++++- asreview/webapp/app.py | 196 ++++++ asreview/webapp/start_flask.py | 566 ------------------ asreview/webapp/tests/conftest.py | 4 +- asreview/webapp/tests/test_api/test_webapp.py | 2 +- .../test_database_creation.py | 11 +- 8 files changed, 576 insertions(+), 689 deletions(-) create mode 100644 asreview/webapp/app.py delete mode 100644 asreview/webapp/start_flask.py diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 324098172..a94810ec1 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -1,48 +1,125 @@ # DEVELOPMENT -## Build project +## Development workflow for frontend and backend development -Build the project from source with the following code. +Most users will only need the first 2 steps: Installation and Setting Up Servers. - python setup.py compile_assets - python setup.py sdist bdist_wheel +### Installation -## Development workflow +Install Python and [Node.js](https://nodejs.org/en) (we use Node v20). -### Git Submodules -Some demo datasets are included as a submodule. Directory [asreview/tests/citation-file-formatting](https://github.com/ottomattas/asreview/tree/development-v1/tests) is cloned from [citation-file-formatting](https://github.com/asreview/citation-file-formatting). +Install ASReview in editable mode -Examples: -- To clone the full repository with submodules in one line, add `--recursive` flag: +```sh +pip install -e .[dev] +``` - ```git clone --recursive git://github.com/asreview/asreview.git``` +Navigate into `asreview/webapp` and install NPM packages -- To update the submodule, you would still need to follow the contribution guide in the submodule repository. And then create a PR for the main repository with the updated submodule commit. +```sh +cd asreview/webapp +npm install +``` -### Back end +### Setting up servers -Install Python +The best development workflow for the ASReview frontend and backend makes use +of 2 simultanously running servers. One serves the Python server with the +Flask app and the other the Node server with the frontend. -Install the ASReview package +Open a command line interface (e.g. Terminal or CMD.exe) and navigate to +`asreview/webapp`. Start the Flask app with -``` -pip install -e .[dev] +```sh +cd asreview/webapp +flask run --debug ``` -Start the Python API server with the Flask development environment +Next, open a new command line interface and navigate to `asreview/webapp`. +Start the local front end application running on a Node server. +```sh +cd asreview/webapp +npm start ``` -export FLASK_DEBUG=1 -asreview lab + +The webbrowser opens at `localhost:3000`. Every time you edit one of the +webapp related Python or Javascript files, the application will automatically +refresh in the browser. + +### Authentication + +When using or developing the authenticated version of ASReview, extra steps +are needed to configure the application. + +Create an authentication config file as instructed in [Authentication] +(#Authentication). Set the environment variable `FLASK_CONFIGFILE` to the +local config file. Start the application again (If Flask app it still running, terminate first) + +```sh +cd asreview/webapp +FLASK_CONFIGFILE=my_config.toml flask run --debug ``` -For Windows, use +The server will read the file and start the authenticated version. +### Advanced + +#### Port and CORS configuration + +In development, when working on the front end, the front- and backend are +strictly separated. It is assumed the Flask app runs on port 5000 and the +React front end on port 3000. Deviating from these ports will lead to +connection or CORS (Cross-Origin Resource Sharing) issues. + +As for CORS issues: it is necessary to precisely define the "allowed origins" +in the backend. These origins must reflect the URL(s) used by the front end +to call the backend. If correctly configured, they are added to the headers +of the backend response, so they can be verified by your browser. If the list +with origin-URLs doesn't provide a URL that corresponds with the URL used in +the original request of the front end, your request is going to fail. + +**Node server running on port other than 3000** + +Set `ALLOWED_ORIGINS` to the url and port of the Node server. E.g., the server +runs on http://localhost:3010: + +```sh +FLASK_ALLOWED_ORIGINS=http://localhost:3010 flask run --debug ``` -set FLASK_DEBUG=1 -asreview lab + +You can also add `ALLOWED_ORIGINS` to your config file or set the environment +variable `FLASK_ALLOWED_ORIGINS`. + +**Flask app running on port other than 5000** + +Set `REACT_APP_API_URL` to the url and port of the Flask API server. E.g., the +server runs on http://localhost:5010: + +```sh +REACT_APP_API_URL=http://localhost:5010 npm start ``` +Alternative is to add this `REACT_APP_API_URL` to the `.env.development` file in the +`/asreview/webapp` folder. Override this config file with a local version +(e.g. `/asreview/webapp/.env.development.local`). More information https://create-react-app.dev/docs/adding-custom-environment-variables/#adding-development-environment-variables-in-env. + +## Testing + + +### Git Submodules +Some demo datasets are included as a submodule. Directory [asreview/tests/citation-file-formatting](https://github.com/ottomattas/asreview/tree/development-v1/tests) is cloned from [citation-file-formatting](https://github.com/asreview/citation-file-formatting). + +Examples: +- To clone the full repository with submodules in one line, add `--recursive` flag: + + ```git clone --recursive git://github.com/asreview/asreview.git``` + +- To update the submodule, you would still need to follow the contribution guide in the submodule repository. And then create a PR for the main repository with the updated submodule commit. + + + + #### Formatting and linting Use `flake8` to lint the Python code and format the code with `black`. Use @@ -63,77 +140,61 @@ isort . flake8 . ``` -### Front end -The user interface is written in [React](https://reactjs.org/). -Please make use of [npx](https://www.npmjs.com/package/npx) and Prettier -(https://prettier.io/docs/en/install.html) to format React/Javascript code. -Afer installing `npx` and `prettier`, navigate to the folder with the file -you want to 'prettify' and run: -``` -npx prettier --write . -``` -To run a local version of the front-end on `localhost:3000`, proceed as -follows: -1. You need to install [Node.js](https://nodejs.org/en) for local development (we use version 20). +## Documentation -2. Before the front end development can be started, the back end has to run as well. Therefore, first, start the Python API server with the Flask development environment: +### Sphinx docs -``` -export FLASK_DEBUG=1 -asreview lab -``` +Documentation for the ASReview project is available on https://asreview.readthedocs.io/en/latest/. +The source files are available in the [`docs`](/docs) folder of this repository. The project makes +use of [Sphinx](https://www.sphinx-doc.org/) to convert the source files and docstrings into HTML +or PDF files. -For Windows, use +Install the dependencies for rendering the documentation with ``` -set FLASK_DEBUG=1 -asreview lab +pip install .[docs] ``` -Note, when working with PowerShell use +Navigate into the `docs` folder and render the documentation (the HTML version) with ``` -$env:FLASK_DEBUG = "1" -asreview lab +make html ``` -**Important**: Ignore `localhost:5000`, because this is not relevant for the - development version, which will run on `localhost:3000`. +Open the file `docs/build/html/index.html` in your web browser. -3. Next, open a new CLI and navigate to `asreview/webapp` and install the front end application with [npm](https://www.npmjs.com/get-npm): +### Broken links + +Navigate into the `docs` folder and check for broken links with: ``` -cd asreview/webapp -npm install +make linkcheck ``` -Start the local front end application with npm +Extra information: https://www.writethedocs.org/guide/tools/testing/#link-testing -``` -npm start -``` +### Screenshots -4. Open the web browser at `localhost:3000` +Screenshots are an important part of the ASReview documentation. When contributing screenshots, +follow the guidelines below. + +1. Open Developers Tools in your browser (e.g. Chrome or Firefox). +2. Set device dimensions to **1280x800**. +3. Capture screenshot with internal screenshot tool (preferred, see [example](https://www.deconetwork.com/blog/how-to-take-full-webpage-screenshots-instantly/)). +4. [OPTIONAL] Crop relevant part. Keep ratio if possible. +5. Resize image to **1280x800** maximum and **960x600** minimum. +6. [OPTIONAL] Use a red box to highlight relevant components. -### Front end development and connection/CORS issues -In development, when working on the front end, the front- and backend are strictly separated. It is assumed the Flask backend runs on port 5000 and the React front end on port 3000. Deviating from these ports will lead to connection or CORS (Cross-Origin Resource Sharing) issues. -As for CORS issues: it is necessary to precisely define the "allowed origins" in the backend. These origins must reflect the URL(s) used by the front end to call the backend. If correctly configured, they are added to the headers of the backend response, so they can be verified by your browser. If the list with origin-URLs doesn't provide a URL that corresponds with the URL used in the original request of the front end, your request is going to fail. __Setting the allowed origins can be done in the [config file](#full-configuration)__. -You can solve connection/CORS issues by doing the following: -1. Start the backend and verify what port number it's running on (read the first lines of the output once you've started the backend in the terminal). -2. Make sure the front end knows where it can find the backend. React reads a configuration `.env` file in the `/asreview/webapp` folder which tells it to use `http://localhost:5000/`. Override this config file by either adding a local version (e.g. `/asreview/webapp/.env.local`) in which you put the correct backend URL (do not forget the `REACT_APP_API_URL` variable, see the `.env` file) or change the URL in the `.env` file itself. -3. If you are running the front end separate from the backend you need to adjust the CORS's 'allowed origins' parameter in the backend to avoid problems. You can do this by setting the front end URL(s) in the [optional parameters of the config file](#optional-config-parameters) under the "ALLOWED_ORIGINS" key. -Be precise when it comes to URLs/port numbers! In the context of CORS `localhost` is different from `127.0.0.1`, although they are normally referring to the same host. -❗Mac users beware: depending on your version of macOS you may experience troubles with `localhost:5000`. Port 5000 may be in use by "Airplay Receiver" which may (!) cause nondeterministic behavior. If you experience similar issues [switch to a different port](#optional-config-parameters). @@ -174,7 +235,6 @@ one could use the User model that can be found in `/asreview/webapp/authenticati To configure the authentication in more detail we need to create a TOML file that contains all authentication parameters. The parameters in that TOML file will override parameters that were passed in the CLI. Here's an example: ```toml -DEBUG = true AUTHENTICATION_ENABLED = true SECRET_KEY = "" SECURITY_PASSWORD_SALT = "" @@ -309,51 +369,6 @@ One can also insert all project information by using the JSON string that was pr $ asreview auth-tool link-projects --json "[{\"folder\": \"project-id\", \"version\": \"1.1+51.g0ebdb0c.dirty\", \"project_id\": \"project-id\", \"name\": \"project 1\", \"authors\": \"Authors\", \"created\": \"2023-04-12 21:23:28.625859\", \"owner_id\": 15}]" --db-path ~/.asreview/asreview.production.sqlite ``` -## Documentation - -### Sphinx docs - -Documentation for the ASReview project is available on https://asreview.readthedocs.io/en/latest/. -The source files are available in the [`docs`](/docs) folder of this repository. The project makes -use of [Sphinx](https://www.sphinx-doc.org/) to convert the source files and docstrings into HTML -or PDF files. - -Install the dependencies for rendering the documentation with - -``` -pip install .[docs] -``` - -Navigate into the `docs` folder and render the documentation (the HTML version) with - -``` -make html -``` - -Open the file `docs/build/html/index.html` in your web browser. - -### Broken links - -Navigate into the `docs` folder and check for broken links with: - -``` -make linkcheck -``` - -Extra information: https://www.writethedocs.org/guide/tools/testing/#link-testing - -### Screenshots - -Screenshots are an important part of the ASReview documentation. When contributing screenshots, -follow the guidelines below. - -1. Open Developers Tools in your browser (e.g. Chrome or Firefox). -2. Set device dimensions to **1280x800**. -3. Capture screenshot with internal screenshot tool (preferred, see [example](https://www.deconetwork.com/blog/how-to-take-full-webpage-screenshots-instantly/)). -4. [OPTIONAL] Crop relevant part. Keep ratio if possible. -5. Resize image to **1280x800** maximum and **960x600** minimum. -6. [OPTIONAL] Use a red box to highlight relevant components. - ## Release instructions diff --git a/Docker/auth_verified/wsgi.py b/Docker/auth_verified/wsgi.py index 99566d205..baf4b33a8 100644 --- a/Docker/auth_verified/wsgi.py +++ b/Docker/auth_verified/wsgi.py @@ -1,3 +1,3 @@ -from asreview.webapp.start_flask import create_app +from asreview.webapp.app import create_app app = create_app() diff --git a/asreview/entry_points/lab.py b/asreview/entry_points/lab.py index adc1b0ce8..d2754064d 100644 --- a/asreview/entry_points/lab.py +++ b/asreview/entry_points/lab.py @@ -11,18 +11,132 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import argparse +import logging +import os +import socket +import webbrowser +from threading import Timer +from gevent.pywsgi import WSGIServer + +from asreview._deprecated import DeprecateAction +from asreview._deprecated import mark_deprecated_help_strings from asreview.entry_points.base import BaseEntryPoint +from asreview.project import ASReviewProject +from asreview.project import get_project_path +from asreview.project import get_projects +from asreview.webapp.app import create_app from asreview.webapp.run_model import main as main_run_model -from asreview.webapp.start_flask import main as main_flask + +# Host name +HOST_NAME = os.getenv("ASREVIEW_HOST") +if HOST_NAME is None: + HOST_NAME = "localhost" + +PORT_NUMBER = 5000 + + +def _deprecated_dev_mode(): + if os.environ.get("FLASK_DEBUG", "") == "1": + print( + "\n\n\n!IMPORTANT!\n\n" + "asreview lab development mode is deprecated, see:\n" + "https://github.com/J535D165/asreview/blob/master/DEVELOPMENT.md" + "\n\n\n" + ) + exit(1) + + +def _check_port_in_use(host, port): + + logging.info(f"Checking if host and port are available :: {host}:{port}") + host = host.replace("https://", "").replace("http://", "") + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + return s.connect_ex((host, port)) == 0 + + +def _open_browser(start_url): + + Timer(1, lambda: webbrowser.open_new(start_url)).start() + + print( + "\n\n\n\nIf your browser doesn't open. " + f"Navigate to {start_url}\n\n\n\n" + ) class LABEntryPoint(BaseEntryPoint): """Entry point to start the ASReview LAB webapp.""" def execute(self, argv): + # check deprecated dev mode + _deprecated_dev_mode() + + parser = _lab_parser() + mark_deprecated_help_strings(parser) + args = parser.parse_args(argv) + + app = create_app( + env="production", + config_file=args.flask_config_file, + secret_key=args.secret_key, + salt=args.salt, + enable_authentication=args.enable_authentication, + ) + app.config["PROPAGATE_EXCEPTIONS"] = False + + # clean all projects + # TODO@{Casper}: this needs a little bit + # of work, we need to access all sub-folders + if args.clean_all_projects: + print("Cleaning all project files.") + for project in get_projects(): + project.clean_tmp_files() + print("Done") + return + + # clean project by project_id + # TODO@{Casper}: cleaning without a user context + # is meaningless -> I think we should remove this + # option + if args.clean_project is not None: + print(f"Cleaning project file '{args.clean_project}'.") + ASReviewProject(get_project_path(args.clean_project)).clean_tmp_files() + print("Done") + return + + # if port is already taken find another one + port = args.port + original_port = port + while _check_port_in_use(args.host, port) is True: + old_port = port + port = int(port) + 1 + if port - original_port >= args.port_retries: + raise ConnectionError( + "Could not find an available port \n" + "to launch ASReview LAB. Last port \n" + f"was {str(port)}" + ) + print(f"Port {old_port} is in use.\n* Trying to start at {port}") + + protocol = "https://" if args.certfile and args.keyfile else "http://" + start_url = f"{protocol}{args.host}:{port}/" + + ssl_args = {} + if args.keyfile and args.certfile: + ssl_args = {"keyfile": args.keyfile, "certfile": args.certfile} + + server = WSGIServer((args.host, port), app, **ssl_args) + print(f"Serving ASReview LAB at {start_url}") - main_flask(argv) + if not args.no_browser: + _open_browser(start_url) + + try: + server.serve_forever() + except KeyboardInterrupt: + print("\n\nShutting down server\n\n") class WebRunModelEntryPoint(BaseEntryPoint): @@ -30,3 +144,134 @@ class WebRunModelEntryPoint(BaseEntryPoint): def execute(self, argv): main_run_model(argv) + + +def _lab_parser(): + # parse arguments if available + parser = argparse.ArgumentParser( + prog="lab", + description="""ASReview LAB - Active learning for Systematic Reviews.""", # noqa + formatter_class=argparse.RawTextHelpFormatter, + ) + + parser.add_argument( + "--clean-project", + dest="clean_project", + default=None, + type=str, + help="Safe cleanup of temporary files in project.", + ) + + parser.add_argument( + "--clean-all-projects", + dest="clean_all_projects", + default=None, + action="store_true", + help="Safe cleanup of temporary files in all projects.", + ) + + parser.add_argument( + "--ip", + default=HOST_NAME, + type=str, + action=DeprecateAction, + help="The IP address the server will listen on. Use the --host argument.", + ) + + parser.add_argument( + "--host", + default=HOST_NAME, + type=str, + help="The IP address the server will listen on.", + ) + + parser.add_argument( + "--port", + default=PORT_NUMBER, + type=int, + help="The port the server will listen on.", + ) + + parser.add_argument( + "--enable-auth", + dest="enable_authentication", + action="store_true", + help="Enable authentication.", + ) + + parser.add_argument( + "--secret-key", + default=None, + type=str, + help="Secret key for authentication.", + ) + + parser.add_argument( + "--salt", + default=None, + type=str, + help="When using authentication, a salt code is needed for hasing passwords.", + ) + + parser.add_argument( + "--flask-configfile", + dest="flask_config_file", + type=str, + help="Full path to a TOML file containing Flask parameters" + "for authentication.", + ) + + parser.add_argument( + "--no-browser", + dest="no_browser", + action="store_true", + help="Do not open ASReview LAB in a browser after startup.", + ) + + parser.add_argument( + "--port-retries", + dest="port_retries", + default=50, + type=int, + help="The number of additional ports to try if the" + "specified port is not available.", + ) + + parser.add_argument( + "--certfile", + default="", + type=str, + help="The full path to an SSL/TLS certificate file.", + ) + + parser.add_argument( + "--keyfile", + default="", + type=str, + help="The full path to a private key file for usage with SSL/TLS.", + ) + + parser.add_argument( + "--config_file", + type=str, + default=None, + help="Deprecated, see subcommand simulate.", + action=DeprecateAction, + ) + + parser.add_argument( + "--seed", + default=None, + type=int, + help="Deprecated, see subcommand simulate.", + action=DeprecateAction, + ) + + parser.add_argument( + "--embedding", + type=str, + default=None, + dest="embedding_fp", + help="File path of embedding matrix. Required for LSTM models.", + ) + return parser diff --git a/asreview/webapp/app.py b/asreview/webapp/app.py new file mode 100644 index 000000000..531d54ba7 --- /dev/null +++ b/asreview/webapp/app.py @@ -0,0 +1,196 @@ +# Copyright 2019-2022 The ASReview Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +from pathlib import Path + +try: + import tomllib +except ImportError: + import tomli as tomllib + +from flask import Flask +from flask import send_from_directory +from flask.json import jsonify +from flask.templating import render_template +from flask_cors import CORS +from flask_login import LoginManager +from werkzeug.exceptions import InternalServerError + +from asreview import __version__ as asreview_version +from asreview.utils import asreview_path +from asreview.webapp import DB +from asreview.webapp.api import auth +from asreview.webapp.api import projects +from asreview.webapp.api import team +from asreview.webapp.authentication.models import User +from asreview.webapp.authentication.oauth_handler import OAuthHandler + + +def create_app( + env="development", + config_file=None, + secret_key=None, + salt=None, + enable_authentication=False, +): + app = Flask( + __name__, + instance_relative_config=True, + static_folder="build/static", + template_folder="build", + ) + + # if app.debug: + app.config["ALLOWED_ORIGINS"] = ["http://localhost:3000", "http://127.0.0.1:3000"] + + app.config["SECRET_KEY"] = secret_key + app.config["SALT"] = salt + app.config["ENABLE_AUTHENTICATION"] = enable_authentication + + app.config.from_prefixed_env() + + if config_file_path := config_file or app.config.get("CONFIGFILE", ""): + app.config.from_file( + Path(config_file_path).absolute(), load=tomllib.load, text=False + ) + + # config JSON Web Tokens + login_manager = LoginManager(app) + login_manager.init_app(app) + login_manager.session_protection = "strong" + + if not app.config.get("AUTHENTICATION_ENABLED", False): + + @login_manager.user_loader + def load_user(user_id): + return False + + elif app.config.get("AUTHENTICATION_ENABLED", False): + # Register a callback function for current_user. + @login_manager.user_loader + def load_user(user_id): + return User.query.get(int(user_id)) + + if app.config.get("EMAIL_VERIFICATION", False) and not app.config.get( + "EMAIL_CONFIG", False + ): + raise ValueError( + "Missing email configuration to facilitate email verification" + ) + + # set email config for Flask-Mail + conf = app.config.get("EMAIL_CONFIG", {}) + app.config["MAIL_SERVER"] = conf.get("SERVER") + app.config["MAIL_PORT"] = conf.get("PORT", 465) + app.config["MAIL_USERNAME"] = conf.get("USERNAME") + app.config["MAIL_PASSWORD"] = conf.get("PASSWORD") + app.config["MAIL_USE_TLS"] = conf.get("USE_TLS", False) + app.config["MAIL_USE_SSL"] = conf.get("USE_SSL", False) + app.config["MAIL_REPLY_ADDRESS"] = conf.get("REPLY_ADDRESS") + + if not app.config.get("SQLALCHEMY_DATABASE_URI", None): + uri = os.path.join(asreview_path(), f"asreview.{env}.sqlite") + app.config["SQLALCHEMY_DATABASE_URI"] = f"sqlite:///{uri}" + + # initialize app for SQLAlchemy + DB.init_app(app) + + with app.app_context(): + # create tables in case they don't exist + DB.create_all() + + # store oauth config in oauth handler + if bool(app.config.get("OAUTH", False)): + app.config["OAUTH"] = OAuthHandler(app.config["OAUTH"]) + + # Ensure the instance folder exists. + try: + os.makedirs(app.instance_path) + except OSError: + pass + + if origins := app.config.get("ALLOWED_ORIGINS", False): + CORS(app, origins=origins, supports_credentials=True) + + with app.app_context(): + app.register_blueprint(projects.bp) + app.register_blueprint(auth.bp) + app.register_blueprint(team.bp) + + @app.errorhandler(InternalServerError) + def error_500(e): + original = getattr(e, "original_exception", None) + + if original is None: + # direct 500 error, such as abort(500) + logging.error(e) + return jsonify(message="Whoops, something went wrong."), 500 + + # wrapped unhandled error + logging.error(e.original_exception) + return jsonify(message=str(e.original_exception)), 500 + + @app.route("/", methods=["GET"]) + @app.route("/confirm_account", methods=["GET"]) + @app.route("/oauth_callback", methods=["GET"]) + @app.route("/projects/", methods=["GET"]) + @app.route("/projects//", methods=["GET"]) + @app.route("/projects///", methods=["GET"]) + @app.route("/reset_password", methods=["GET"]) + def index(): + return render_template("index.html") + + @app.route("/favicon.ico") + def send_favicon(): + return send_from_directory( + "build", "favicon.ico", mimetype="image/vnd.microsoft.icon" + ) + + @app.route("/boot", methods=["GET"]) + def api_boot(): + """Get the boot info.""" + + authenticated = app.config.get("AUTHENTICATION_ENABLED", False) + + response = { + "authentication": authenticated, + "version": asreview_version, + } + + if authenticated: + # if recaptcha config is provided for account creation + if app.config.get("RE_CAPTCHA_V3", False): + response["recaptchav3_key"] = app.config["RE_CAPTCHA_V3"].get( + "KEY", False + ) + + response["allow_account_creation"] = app.config.get( + "ALLOW_ACCOUNT_CREATION", False + ) + response["allow_teams"] = app.config.get("ALLOW_TEAMS", False) + response["email_verification"] = bool( + app.config.get("EMAIL_VERIFICATION", False) + ) + response["email_config"] = bool(app.config.get("EMAIL_CONFIG", False)) + + # if oauth config is provided + if isinstance(app.config.get("OAUTH", False), OAuthHandler): + if params := app.config.get("OAUTH").front_end_params(): + response["oauth"] = params + + return jsonify(response) + + return app diff --git a/asreview/webapp/start_flask.py b/asreview/webapp/start_flask.py deleted file mode 100644 index b9096b888..000000000 --- a/asreview/webapp/start_flask.py +++ /dev/null @@ -1,566 +0,0 @@ -# Copyright 2019-2022 The ASReview Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import logging -import os -import socket -import webbrowser -from pathlib import Path -from threading import Timer - -try: - import tomllib -except ImportError: - import tomli as tomllib - -from flask import Flask -from flask import send_from_directory -from flask.json import jsonify -from flask.templating import render_template -from flask_cors import CORS -from flask_login import LoginManager -from gevent.pywsgi import WSGIServer -from werkzeug.exceptions import InternalServerError - -from asreview import __version__ as asreview_version -from asreview._deprecated import DeprecateAction -from asreview._deprecated import mark_deprecated_help_strings -from asreview.project import ASReviewProject -from asreview.project import get_project_path -from asreview.project import get_projects -from asreview.utils import asreview_path -from asreview.webapp import DB -from asreview.webapp.api import auth -from asreview.webapp.api import projects -from asreview.webapp.api import team -from asreview.webapp.authentication.models import User -from asreview.webapp.authentication.oauth_handler import OAuthHandler - -# Host name -HOST_NAME = os.getenv("ASREVIEW_HOST") -if HOST_NAME is None: - HOST_NAME = "localhost" -# Default Port number -PORT_NUMBER = 5000 - -# set logging level -if ( - os.environ.get("FLASK_DEBUG", "") == "1" - or os.environ.get("DEBUG", "") == "1" - or os.environ.get("FLASK_ENV", "") == "development" -): - logging.basicConfig(level=logging.DEBUG) -else: - logging.basicConfig(level=logging.INFO) - - -def _url(host, port, protocol): - """Create url from host and port.""" - return f"{protocol}{host}:{port}/" - - -def _check_port_in_use(host, port): - """Check if port is already in use. - - Arguments - --------- - host: str - The current host. - port: int - The host port to be checked. - - Returns - ------- - bool: - True if port is in use, false otherwise. - """ - logging.info(f"Checking if host and port are available :: {host}:{port}") - host = host.replace("https://", "").replace("http://", "") - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - return s.connect_ex((host, port)) == 0 - - -def _open_browser(host, port, protocol, no_browser): - """Open ASReview in browser if flag is set. - - Otherwise, it displays an alert to copy and paste the url - at which ASReview is currently served. - """ - if no_browser: - print( - "\nTo access ASReview LAB, copy and paste " - "this url in a browser " - f"{_url(host, port, protocol)}\n" - ) - return - - start_url = _url(host, port, protocol) - Timer(1, lambda: webbrowser.open_new(start_url)).start() - print( - f"Start browser at {start_url}" - "\n\n\n\nIf your browser doesn't open. " - f"Please navigate to '{start_url}'\n\n\n\n" - ) - - -def _lab_parser(): - # parse arguments if available - parser = argparse.ArgumentParser( - prog="lab", - description="""ASReview LAB - Active learning for Systematic Reviews.""", # noqa - formatter_class=argparse.RawTextHelpFormatter, - ) - - parser.add_argument( - "--clean-project", - dest="clean_project", - default=None, - type=str, - help="Safe cleanup of temporary files in project.", - ) - - parser.add_argument( - "--clean-all-projects", - dest="clean_all_projects", - default=None, - action="store_true", - help="Safe cleanup of temporary files in all projects.", - ) - - parser.add_argument( - "--ip", - default=HOST_NAME, - type=str, - action=DeprecateAction, - help="The IP address the server will listen on. Use the --host argument.", - ) - - parser.add_argument( - "--host", - default=HOST_NAME, - type=str, - help="The IP address the server will listen on.", - ) - - parser.add_argument( - "--port", - default=PORT_NUMBER, - type=int, - help="The port the server will listen on.", - ) - - parser.add_argument( - "--enable-auth", - dest="enable_authentication", - action="store_true", - help="Enable authentication.", - ) - - parser.add_argument( - "--auth-database-uri", - default=None, - type=str, - help="URI of authentication database.", - ) - - parser.add_argument( - "--secret-key", - default=None, - type=str, - help="Secret key for authentication.", - ) - - parser.add_argument( - "--salt", - default=None, - type=str, - help="When using authentication, a salt code is needed" "for hasing passwords.", - ) - - parser.add_argument( - "--flask-configfile", - default="", - type=str, - help="Full path to a TOML file containing Flask parameters" - "for authentication.", - ) - - parser.add_argument( - "--no-browser", - dest="no_browser", - action="store_true", - help="Do not open ASReview LAB in a browser after startup.", - ) - - parser.add_argument( - "--port-retries", - dest="port_retries", - default=50, - type=int, - help="The number of additional ports to try if the" - "specified port is not available.", - ) - - parser.add_argument( - "--certfile", - default="", - type=str, - help="The full path to an SSL/TLS certificate file.", - ) - - parser.add_argument( - "--keyfile", - default="", - type=str, - help="The full path to a private key file for usage with SSL/TLS.", - ) - - parser.add_argument( - "--config_file", - type=str, - default=None, - help="Deprecated, see subcommand simulate.", - action=DeprecateAction, - ) - - parser.add_argument( - "--seed", - default=None, - type=int, - help="Deprecated, see subcommand simulate.", - action=DeprecateAction, - ) - - parser.add_argument( - "--embedding", - type=str, - default=None, - dest="embedding_fp", - help="File path of embedding matrix. Required for LSTM models.", - ) - return parser - - -def create_app(**kwargs): - app = Flask( - __name__, - instance_relative_config=True, - static_folder="build/static", - template_folder="build", - ) - - # Get the ASReview arguments. - app.config["asr_kwargs"] = kwargs - app.config["AUTHENTICATION_ENABLED"] = kwargs.get("enable_authentication", False) - app.config["SECRET_KEY"] = kwargs.get("secret_key", False) - app.config["SECURITY_PASSWORD_SALT"] = kwargs.get("salt", False) - app.config["PORT"] = kwargs.get("port") - app.config["HOST"] = kwargs.get("host") - - # Read config parameters if possible, this overrides - # the previous assignments. Flask config parameters may come - # as an environment var or from an argument. Argument - # takes precedence. - config_from_env = os.environ.get("FLASK_CONFIGFILE", "").strip() - config_from_arg = kwargs.get("flask_configfile", "").strip() - config_file_path = config_from_arg or config_from_env - - # Use absolute path, because otherwise it is relative to the config root. - if config_file_path != "": - config_file_path = Path(config_file_path) - if config_file_path.suffix == ".toml": - app.config.from_file( - config_file_path.absolute(), - load=tomllib.load, - text=False - ) - else: - raise ValueError("'flask_configfile' should have a .toml extension") - - # If the frontend runs on a different port, or even on a different - # URL, then allowed-origins must be set to avoid CORS issues. You can - # set the allowed-origins in the config file. In the previous lines - # the config file has been read. - # If the allowed-origins are not set by now, they are set to - # False, which will bypass setting any CORS parameters! - if not app.config.get("ALLOWED_ORIGINS", False): - app.config["ALLOWED_ORIGINS"] = False - - if not app.config["ALLOWED_ORIGINS"] and os.environ.get("FLASK_DEBUG", "") == "1": - app.config["ALLOWED_ORIGINS"] = [f"http://{app.config['HOST']}:3000"] - - # set env (test / development / production) according to - # Flask 2.2 specs (ENV is deprecated) - if app.config.get("TESTING", None) is True: - env = "test" - elif app.config.get("DEBUG", None) is True: - env = "development" - else: - env = "production" - - # config JSON Web Tokens - login_manager = LoginManager(app) - login_manager.init_app(app) - login_manager.session_protection = "strong" - - if app.config["AUTHENTICATION_ENABLED"] is False: - # This ensures the app handles the anonymous user - # when authentication is disabled and there is no - # configuration file - app.config["SECRET_KEY"] = "" - - # This is necessary to pass the test_webapp.py tests - @login_manager.user_loader - def load_user(user_id): - return False - - # setup all database/authentication related resources, - # only do this when AUTHENTICATION_ENABLED is explicitly True - elif app.config["AUTHENTICATION_ENABLED"] is True: - # Register a callback function for current_user. - @login_manager.user_loader - def load_user(user_id): - return User.query.get(int(user_id)) - - # In this code-block we make sure certain authentication-related - # config parameters are set. - # TODO: should I raise a custom Exception, like MissingParameterError? - if not app.config.get("SECRET_KEY", False): - raise ValueError( - "Please start an authenticated app with a " - + "secret key parameter (SECRET_KEY)" - ) - - if not app.config.get("SECURITY_PASSWORD_SALT", False): - raise ValueError( - "Please start an authenticated app with a " - + "security password salt (SECURITY_PASSWORD_SALT)" - ) - - if app.config.get("EMAIL_VERIFICATION", False) and not app.config.get( - "EMAIL_CONFIG", False - ): - raise ValueError( - "Missing email configuration to facilitate email verification" - ) - - # set email config for Flask-Mail - conf = app.config.get("EMAIL_CONFIG", {}) - app.config["MAIL_SERVER"] = conf.get("SERVER") - app.config["MAIL_PORT"] = conf.get("PORT", 465) - app.config["MAIL_USERNAME"] = conf.get("USERNAME") - app.config["MAIL_PASSWORD"] = conf.get("PASSWORD") - app.config["MAIL_USE_TLS"] = conf.get("USE_TLS", False) - app.config["MAIL_USE_SSL"] = conf.get("USE_SSL", False) - app.config["MAIL_REPLY_ADDRESS"] = conf.get("REPLY_ADDRESS") - - # We must be sure we have a SQLAlchemy database URI. At this - # stage the TOML file has been read. See if we haven't found - # such a URI. - if not app.config.get("SQLALCHEMY_DATABASE_URI", False): - # there is no configuration, check CLI parameters - cli_database_uri = (kwargs.get("auth_database_uri") or "").strip() - - # if we still haven't found a database URI, create a sqlite3 database - if cli_database_uri != "": - app.config["SQLALCHEMY_DATABASE_URI"] = cli_database_uri - else: - # create default path - uri = os.path.join(asreview_path(), f"asreview.{env}.sqlite") - app.config["SQLALCHEMY_DATABASE_URI"] = f"sqlite:///{uri}" - - # initialize app for SQLAlchemy - DB.init_app(app) - - with app.app_context(): - # create tables in case they don't exist - DB.create_all() - - # store oauth config in oauth handler - if bool(app.config.get("OAUTH", False)): - app.config["OAUTH"] = OAuthHandler(app.config["OAUTH"]) - - # Ensure the instance folder exists. - try: - os.makedirs(app.instance_path) - except OSError: - pass - - # We only need CORS if they are necessary: when the frontend is - # running on a different port, or even url, we need to set the - # allowed origins to avoid CORS problems. The allowed-origins - # can be set in the config file. - if app.config.get("ALLOWED_ORIGINS", False): - CORS(app, origins=app.config.get("ALLOWED_ORIGINS"), supports_credentials=True) - - with app.app_context(): - app.register_blueprint(projects.bp) - app.register_blueprint(auth.bp) - app.register_blueprint(team.bp) - - @app.errorhandler(InternalServerError) - def error_500(e): - original = getattr(e, "original_exception", None) - - if original is None: - # direct 500 error, such as abort(500) - logging.error(e) - return jsonify(message="Whoops, something went wrong."), 500 - - # wrapped unhandled error - logging.error(e.original_exception) - return jsonify(message=str(e.original_exception)), 500 - - @app.route("/", methods=["GET"]) - @app.route("/confirm_account", methods=["GET"]) - @app.route("/oauth_callback", methods=["GET"]) - @app.route("/projects/", methods=["GET"]) - @app.route("/projects//", methods=["GET"]) - @app.route("/projects///", methods=["GET"]) - @app.route("/reset_password", methods=["GET"]) - def index(**kwargs): - return render_template("index.html") - - @app.route("/favicon.ico") - def send_favicon(): - return send_from_directory( - "build", "favicon.ico", mimetype="image/vnd.microsoft.icon" - ) - - @app.route("/boot", methods=["GET"]) - def api_boot(): - """Get the boot info.""" - if app.config.get("DEBUG", None) is True: - status = "development" - else: - status = "asreview" - - # the big one - authenticated = app.config.get("AUTHENTICATION_ENABLED", False) - - response = { - "status": status, - "authentication": authenticated, - "version": asreview_version, - } - - # if we do authentication we have a lot of extra parameters - if authenticated: - # if recaptcha config is provided for account creation - if app.config.get("RE_CAPTCHA_V3", False): - response["recaptchav3_key"] = app.config["RE_CAPTCHA_V3"].get( - "KEY", False - ) - - # check if users can create accounts - response["allow_account_creation"] = app.config.get( - "ALLOW_ACCOUNT_CREATION", False - ) - - response["allow_teams"] = app.config.get("ALLOW_TEAMS", False) - - # check if we are doing email verification - response["email_verification"] = bool( - app.config.get("EMAIL_VERIFICATION", False) - ) - - # check if there is an email server setup (forgot password) - response["email_config"] = bool(app.config.get("EMAIL_CONFIG", False)) - - # if oauth config is provided - if isinstance(app.config.get("OAUTH", False), OAuthHandler): - params = app.config.get("OAUTH").front_end_params() - # and there something in it, just to be sure - if params: - response["oauth"] = params - - return jsonify(response) - - return app - - -def main(argv): - parser = _lab_parser() - mark_deprecated_help_strings(parser) - args = parser.parse_args(argv) - - app = create_app(**vars(args)) - app.config["PROPAGATE_EXCEPTIONS"] = False - - # ssl certificate, key and protocol - certfile = args.certfile - keyfile = args.keyfile - ssl_context = None - if certfile and keyfile: - protocol = "https://" - ssl_context = (certfile, keyfile) - else: - protocol = "http://" - - # clean all projects - # TODO@{Casper}: this needs a little bit - # of work, we need to access all sub-folders - if args.clean_all_projects: - print("Cleaning all project files.") - for project in get_projects(): - project.clean_tmp_files() - print("Done") - return - - # clean project by project_id - # TODO@{Casper}: cleaning without a user context - # is meaningless -> I think we should remove this - # option - if args.clean_project is not None: - print(f"Cleaning project file '{args.clean_project}'.") - ASReviewProject(get_project_path(args.clean_project)).clean_tmp_files() - print("Done") - return - - flask_dev = app.config.get("DEBUG", False) - - host = app.config.get("HOST") - port = app.config.get("PORT") - - port_retries = args.port_retries - # if port is already taken find another one - if not flask_dev: - original_port = port - while _check_port_in_use(host, port) is True: - old_port = port - port = int(port) + 1 - if port - original_port >= port_retries: - raise ConnectionError( - "Could not find an available port \n" - "to launch ASReview LAB. Last port \n" - f"was {str(port)}" - ) - print(f"Port {old_port} is in use.\n* Trying to start at {port}") - - # open webbrowser if not in flask development mode - if flask_dev is False: - _open_browser(host, port, protocol, args.no_browser) - - # run app in flask mode only if we run in development mode - if flask_dev is True: - app.run(host=host, port=port, ssl_context=ssl_context) - else: - ssl_args = {"keyfile": keyfile, "certfile": certfile} if ssl_context else {} - server = WSGIServer((host, port), app, **ssl_args) - - try: - server.serve_forever() - except KeyboardInterrupt: - print("\n\nShutting down server\n\n") diff --git a/asreview/webapp/tests/conftest.py b/asreview/webapp/tests/conftest.py index e21f74942..14749a335 100644 --- a/asreview/webapp/tests/conftest.py +++ b/asreview/webapp/tests/conftest.py @@ -19,7 +19,7 @@ from sqlalchemy.orm import close_all_sessions from asreview.webapp import DB -from asreview.webapp.start_flask import create_app +from asreview.webapp.app import create_app from asreview.webapp.tests.utils import crud PROJECTS = [ @@ -55,7 +55,7 @@ def _get_app(app_type="auth-basic", path=None): else: raise ValueError(f"Unknown config {app_type}") # create app - app = create_app(flask_configfile=config_path) + app = create_app(env="test", config_file=config_path) # and return it return app diff --git a/asreview/webapp/tests/test_api/test_webapp.py b/asreview/webapp/tests/test_api/test_webapp.py index f27948141..ef88a7b32 100644 --- a/asreview/webapp/tests/test_api/test_webapp.py +++ b/asreview/webapp/tests/test_api/test_webapp.py @@ -24,7 +24,7 @@ def test_boot(setup_all_clients): assert status_code == 200 assert isinstance(data, dict) assert "authentication" in data.keys() - assert "status" in data.keys() + # assert "status" in data.keys() # what is the aim of this? assert "version" in data.keys() if misc.current_app_is_authenticated(): assert data["authentication"] diff --git a/asreview/webapp/tests/test_database_and_models/test_database_creation.py b/asreview/webapp/tests/test_database_and_models/test_database_creation.py index 578732015..0cde6cdb3 100644 --- a/asreview/webapp/tests/test_database_and_models/test_database_creation.py +++ b/asreview/webapp/tests/test_database_and_models/test_database_creation.py @@ -1,6 +1,3 @@ -# GOAL: test database creation if app is started with authentication -from pathlib import Path - import pytest import sqlalchemy from sqlalchemy import create_engine @@ -9,20 +6,20 @@ def get_db_path(): - return Path(asreview_path() / "asreview.test.sqlite") + return asreview_path() / "asreview.test.sqlite" # checks if asreview path does not contain a database if app # is unauthenticated def test_database_is_not_created_if_unauth_app(unauth_app): - assert Path(asreview_path()).exists() + assert asreview_path().exists() assert get_db_path().exists() is False # checks is asreview path contains database if app is # authenticated def test_database_exists_after_starting_auth_app(auth_app): - assert Path(asreview_path()).exists() + assert asreview_path().exists() assert get_db_path().exists() @@ -31,6 +28,6 @@ def test_database_exists_after_starting_auth_app(auth_app): "table", ["collaboration_invitations", "collaborations", "projects", "users"] ) def test_if_db_table_exists(auth_app, table): - engine = create_engine(f"sqlite:///{str(get_db_path())}") + engine = create_engine(f"sqlite:///{get_db_path()}") table_names = sqlalchemy.inspect(engine).get_table_names() assert table in table_names