From 5d0c01ce8cefea00d2013900ea15dc011544572a Mon Sep 17 00:00:00 2001 From: Loren Johnson Date: Mon, 15 Jul 2024 10:16:55 +0200 Subject: [PATCH 1/2] WDQS Updater issue fix correction, add Wikibase Port, etc --- build/WDQS/Dockerfile | 6 +- build/WDQS/README.md | 9 +- build/WDQS/entrypoint.sh | 10 --- build/WDQS/runUpdate.sh | 5 +- build/WDQS/wait-for-it.sh | 176 -------------------------------------- 5 files changed, 11 insertions(+), 195 deletions(-) delete mode 100755 build/WDQS/wait-for-it.sh diff --git a/build/WDQS/Dockerfile b/build/WDQS/Dockerfile index 89e9b4436..ec82479ee 100644 --- a/build/WDQS/Dockerfile +++ b/build/WDQS/Dockerfile @@ -61,18 +61,20 @@ RUN mkdir /var/log/wdqs && chown blazegraph /var/log/wdqs ENV MEMORY=""\ HEAP_SIZE="1g"\ HOST="0.0.0.0"\ + WDQS_ENTITY_NAMESPACES="120,122"\ + WDQS_SCHEME="http"\ WDQS_HOST="wdqs"\ WDQS_PORT="9999"\ - WDQS_ENTITY_NAMESPACES="120,122"\ WIKIBASE_SCHEME="http"\ WIKIBASE_HOST="wikibase"\ + WIKIBASE_PORT="80"\ WIKIBASE_MAX_DAYS_BACK="90"\ BLAZEGRAPH_EXTRA_OPTS=""\ LOG_CONFIG="/wdqs/logback.xml" WORKDIR /wdqs -COPY --chown=blazegraph:blazegraph wait-for-it.sh entrypoint.sh runBlazegraph.sh runUpdate.sh / +COPY --chown=blazegraph:blazegraph entrypoint.sh runBlazegraph.sh runUpdate.sh / COPY --chown=blazegraph:blazegraph mwservices.json /templates/mwservices.json COPY --chown=blazegraph:blazegraph RWStore.properties allowlist.txt logback.xml /wdqs/ diff --git a/build/WDQS/README.md b/build/WDQS/README.md index 3c00d20dd..99819bbc3 100644 --- a/build/WDQS/README.md +++ b/build/WDQS/README.md @@ -39,11 +39,13 @@ Variables in **bold** are required. | Variable | Default | Description | | ------------------------ | ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **`WIKIBASE_HOST`** | "wikibase" | Hostname of the Wikibase service | +| `WDQS_ENTITY_NAMESPACES` | "120,122" | Wikibase namespaces to load data from | +| **`WDQS_SCHEME`** | "http" | WDQS scheme (this service) | | **`WDQS_HOST`** | "wdqs" | WDQS hostname (this service) | | **`WDQS_PORT`** | "9999" | WDQS port (this service) | | `WIKIBASE_SCHEME` | "http" | URL scheme of the Wikibase service | -| `WDQS_ENTITY_NAMESPACES` | "120,122" | Wikibase namespaces to load data from | +| **`WIKIBASE_HOST`** | "wikibase" | Hostname of the Wikibase service | +| **`WIKIBASE_PORT`** | "80" | Port of the Wikibase service | | `WIKIBASE_MAX_DAYS_BACK` | "90" | Maximum number of days updater can reach back in time from now | | `MEMORY` | "" | Memory limit for Blazegraph | | `HEAP_SIZE` | "1g" | Heap size for Blazegraph | @@ -206,7 +208,8 @@ In the Docker Compose example provided above, you might run: ```sh docker compose stop wdqs-updater -docker compose run --rm wdqs-updater bash -c '/wdqs/runUpdate.sh -h http://"$WDQS_HOST":"$WDQS_PORT" -- --wikibaseUrl "$WIKIBASE_SCHEME"://"$WIKIBASE_HOST" --conceptUri "$WIKIBASE_SCHEME"://"$WIKIBASE_HOST" --entityNamespaces "$WDQS_ENTITY_NAMESPACES" --init --start $(date +%Y%m%d000000)' +docker compose run --rm wdqs-updater bash -c "echo; /wdqs/runUpdate.sh -h \"\${WDQS_SCHEME}://\${WDQS_HOST}:\${WDQS_PORT}\" -- --wikibaseUrl \"\${WIKIBASE_SCHEME}://\${WIKIBASE_HOST}:\${WIKIBASE_PORT}\" --conceptUri \"\${WIKIBASE_SCHEME}://\${WIKIBASE_HOST}:\${WIKIBASE_PORT}\" --entityNamespaces \"\${WDQS_ENTITY_NAMESPACES}\" --init --start \$(date +%Y%m%d000000)" +# Ctrl-C to cancel once "Sleeping for 10 secs" is seen docker compose start wdqs-updater ``` diff --git a/build/WDQS/entrypoint.sh b/build/WDQS/entrypoint.sh index 5a174ecb3..2088054aa 100755 --- a/build/WDQS/entrypoint.sh +++ b/build/WDQS/entrypoint.sh @@ -1,16 +1,6 @@ #!/usr/bin/env bash # This file is provided by the wikibase/wdqs docker image. -# Test if required environment variables have been set -REQUIRED_VARIABLES=(WIKIBASE_HOST WDQS_HOST WDQS_PORT) -for i in "${REQUIRED_VARIABLES[@]}"; do - eval THISSHOULDBESET=\$"$i" - if [ -z "$THISSHOULDBESET" ]; then - echo "$i is required but isn't set. You should pass it to docker. See: https://docs.docker.com/engine/reference/commandline/run/#set-environment-variables--e---env---env-file"; - exit 1; - fi -done - set -eu export BLAZEGRAPH_OPTS="${BLAZEGRAPH_EXTRA_OPTS} -DwikibaseHost=${WIKIBASE_HOST}" diff --git a/build/WDQS/runUpdate.sh b/build/WDQS/runUpdate.sh index 6eb75472d..50ca44219 100755 --- a/build/WDQS/runUpdate.sh +++ b/build/WDQS/runUpdate.sh @@ -3,7 +3,4 @@ cd /wdqs || exit -# TODO env vars for entity namespaces, scheme and other settings -/wait-for-it.sh "$WIKIBASE_HOST:80" -t 300 -- \ -/wait-for-it.sh "$WDQS_HOST:$WDQS_PORT" -t 300 -- \ -./runUpdate.sh -h http://"$WDQS_HOST":"$WDQS_PORT" -- --wikibaseUrl "$WIKIBASE_SCHEME"://"$WIKIBASE_HOST" --conceptUri "$WIKIBASE_SCHEME"://"$WIKIBASE_HOST" --entityNamespaces "$WDQS_ENTITY_NAMESPACES" +./runUpdate.sh -h "$WDQS_SCHEME"://"$WDQS_HOST":"$WDQS_PORT" -- --wikibaseUrl "$WIKIBASE_SCHEME"://"$WIKIBASE_HOST":"$WIKIBASE_PORT" --conceptUri "$WIKIBASE_SCHEME"://"$WIKIBASE_HOST":"$WIKIBASE_PORT" --entityNamespaces "$WDQS_ENTITY_NAMESPACES" diff --git a/build/WDQS/wait-for-it.sh b/build/WDQS/wait-for-it.sh deleted file mode 100755 index 56be64af1..000000000 --- a/build/WDQS/wait-for-it.sh +++ /dev/null @@ -1,176 +0,0 @@ -#!/usr/bin/env bash -# Use this script to test if a given TCP host/port are available -# shellcheck disable=SC2064,SC2086,SC2206,SC2124,SC2230,SC2046 -cmdname=$(basename $0) - -echoerr() { if [[ $QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } - -usage() -{ - cat << USAGE >&2 -Usage: - $cmdname host:port [-s] [-t timeout] [-- command args] - -h HOST | --host=HOST Host or IP under test - -p PORT | --port=PORT TCP port under test - Alternatively, you specify the host and port as host:port - -s | --strict Only execute subcommand if the test succeeds - -q | --quiet Don't output any status messages - -t TIMEOUT | --timeout=TIMEOUT - Timeout in seconds, zero for no timeout - -- COMMAND ARGS Execute command with args after the test finishes -USAGE - exit 1 -} - -wait_for() -{ - if [[ $TIMEOUT -gt 0 ]]; then - echoerr "$cmdname: waiting $TIMEOUT seconds for $HOST:$PORT" - else - echoerr "$cmdname: waiting for $HOST:$PORT without a timeout" - fi - start_ts=$(date +%s) - while : - do - if [[ $ISBUSY -eq 1 ]]; then - nc -z $HOST $PORT - result=$? - else - (echo > /dev/tcp/$HOST/$PORT) >/dev/null 2>&1 - result=$? - fi - if [[ $result -eq 0 ]]; then - end_ts=$(date +%s) - echoerr "$cmdname: $HOST:$PORT is available after $((end_ts - start_ts)) seconds" - break - fi - sleep 1 - done - return $result -} - -wait_for_wrapper() -{ - # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 - if [[ $QUIET -eq 1 ]]; then - timeout $BUSYTIMEFLAG $TIMEOUT $0 --quiet --child --host=$HOST --port=$PORT --timeout=$TIMEOUT & - else - timeout $BUSYTIMEFLAG $TIMEOUT $0 --child --host=$HOST --port=$PORT --timeout=$TIMEOUT & - fi - PID=$! - trap "kill -INT -$PID" INT - wait $PID - RESULT=$? - if [[ $RESULT -ne 0 ]]; then - echoerr "$cmdname: timeout occurred after waiting $TIMEOUT seconds for $HOST:$PORT" - fi - return $RESULT -} - -# process arguments -while [[ $# -gt 0 ]] -do - case "$1" in - *:* ) - hostport=(${1//:/ }) - HOST=${hostport[0]} - PORT=${hostport[1]} - shift 1 - ;; - --child) - CHILD=1 - shift 1 - ;; - -q | --quiet) - QUIET=1 - shift 1 - ;; - -s | --strict) - STRICT=1 - shift 1 - ;; - -h) - HOST="$2" - if [[ $HOST == "" ]]; then break; fi - shift 2 - ;; - --host=*) - HOST="${1#*=}" - shift 1 - ;; - -p) - PORT="$2" - if [[ $PORT == "" ]]; then break; fi - shift 2 - ;; - --port=*) - PORT="${1#*=}" - shift 1 - ;; - -t) - TIMEOUT="$2" - if [[ $TIMEOUT == "" ]]; then break; fi - shift 2 - ;; - --timeout=*) - TIMEOUT="${1#*=}" - shift 1 - ;; - --) - shift - CLI="$@" - break - ;; - --help) - usage - ;; - *) - echoerr "Unknown argument: $1" - usage - ;; - esac -done - -if [[ "$HOST" == "" || "$PORT" == "" ]]; then - echoerr "Error: you need to provide a host and port to test." - usage -fi - -TIMEOUT=${TIMEOUT:-15} -STRICT=${STRICT:-0} -CHILD=${CHILD:-0} -QUIET=${QUIET:-0} - -# check to see if timeout is from busybox? -TIMEOUT_PATH=$(realpath $(which timeout)) -if [[ $TIMEOUT_PATH =~ "busybox" ]]; then - ISBUSY=1 - BUSYTIMEFLAG="-t" -else - ISBUSY=0 - BUSYTIMEFLAG="" -fi - -if [[ $CHILD -gt 0 ]]; then - wait_for - RESULT=$? - exit $RESULT -else - if [[ $TIMEOUT -gt 0 ]]; then - wait_for_wrapper - RESULT=$? - else - wait_for - RESULT=$? - fi -fi - -if [[ $CLI != "" ]]; then - if [[ $RESULT -ne 0 && $STRICT -eq 1 ]]; then - echoerr "$cmdname: strict mode, refusing to execute subprocess" - exit $RESULT - fi - exec $CLI -else - exit $RESULT -fi From aa24c8f9838f8a6566fe0ad0836f86016cf77540 Mon Sep 17 00:00:00 2001 From: Loren Johnson Date: Wed, 17 Jul 2024 21:46:58 +0200 Subject: [PATCH 2/2] Update README --- build/WDQS/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/WDQS/README.md b/build/WDQS/README.md index 22d76b3ef..5e0c122c0 100644 --- a/build/WDQS/README.md +++ b/build/WDQS/README.md @@ -208,7 +208,7 @@ In the Docker Compose example provided above, you might run: ```sh docker compose stop wdqs-updater -docker compose run --rm wdqs-updater bash '/wdqs/runUpdate.sh -h http://"$WDQS_HOST":"$WDQS_PORT" -- --wikibaseUrl "$WIKIBASE_SCHEME"://"$WIKIBASE_HOST" --conceptUri "$WIKIBASE_SCHEME"://"$WIKIBASE_HOST" --entityNamespaces "$WDQS_ENTITY_NAMESPACES" --init --start $(date +%Y%m%d000000)' +docker compose run --rm wdqs-updater bash '/wdqs/runUpdate.sh -h http://"$WDQS_HOST":"$WDQS_PORT" -- --wikibaseUrl "$WIKIBASE_SCHEME"://"$WIKIBASE_HOST":"$WIKIBASE_PORT" --conceptUri "$WIKIBASE_SCHEME"://"$WIKIBASE_HOST":"$WIKIBASE_PORT" --entityNamespaces "$WDQS_ENTITY_NAMESPACES" --init --start $(date +%Y%m%d000000)' # Ctrl-C to cancel once "Sleeping for 10 secs" is seen docker compose start wdqs-updater ```