Migrate to Java >= 21

.github/workflows/gradle.yml

@@ -11,16 +11,32 @@ on:
 
 jobs:
   build:
-
+    name: Build and analyze
     runs-on: ubuntu-latest
-
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up JDK 1.8
-      uses: actions/setup-java@v1
-      with:
-        java-version: 1.8
-    - name: Grant execute permission for gradlew
-      run: chmod +x gradlew
-    - name: Build with Gradle
-      run: ./gradlew build
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
+      - name: Set up JDK 23
+        uses: actions/setup-java@v4
+        with:
+          java-version: 23
+          distribution: 'temurin' # Alternative distribution options are available
+      - name: Cache SonarQube packages
+        uses: actions/cache@v4
+        with:
+          path: ~/.sonar/cache
+          key: ${{ runner.os }}-sonar
+          restore-keys: ${{ runner.os }}-sonar
+      - name: Grant execute permission for gradlew
+        run: chmod +x gradlew
+      - name: Cache Gradle packages
+        uses: actions/cache@v4
+        with:
+          path: ~/.gradle/caches
+          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
+          restore-keys: ${{ runner.os }}-gradle
+      - name: Build and analyze
+        env:
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+        run: ./gradlew build jacocoTestReport sonar --info

CREST_DOC.md

@@ -0,0 +1,105 @@
+![Java CI with Gradle](https://github.com/HSF/Crest/workflows/Java%20CI%20with%20Gradle/badge.svg?event=push)
+
+#### Author: A.Formica, R.Sipos
+#### Contributors: M.Mineev, E.Alexandrov (client tools)
+```
+   Copyright (C) 2016  A.Formica, R.Sipos
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+```
+# Table of Contents
+1. [Description](#description)
+2. [Data Model](#data-model-overview)
+1. [Workflows](#workflows)
+
+## Description
+CREST service is a RESTful API for the storage and retrieval of IOVs (Interval of Validity) and payloads.
+The data model is illustrated in the following diagram:
+
+```mermaid
+classDiagram
+    GlobalTag --* Maps
+    Tag --* Maps
+    Tag --* Iov
+    Iov *--|> Payload
+    class GlobalTag{
+        +String name
+        +String workflow
+    } 
+    class Tag{
+        +String name
+        +Long endTime
+    } 
+    class Iov{
+        +Long since
+        +String tagName
+        +Timestamp insertionTime
+        +String payloadHash
+    } 
+    class Maps{
+        +String globalTagName
+        +String tagName
+        +String label
+        +String record
+    } 
+    class Payload{
+        +String hash
+        +binary data
+        +String objectType
+    } 
+```
+
+The entity relationship in the relational DB is represented in the following diagram:
+
+```mermaid
+erDiagram
+    GlobalTag ||..|{ GlobalTagMaps : maps
+    GlobalTagMaps }|..|| Tag: maps
+    Tag ||..|| Meta: description
+    Tag ||..o{ Iov : links
+    Iov }|--|| Payload : has
+    Payload ||..|| Data : contains
+    Payload ||..|| Streamer : contains 
+```
+## Data Model overview
+1. `TAG` : a tag is a virtual container of metadata (the `IOVs`), and is defined via the following properties
+    * `name` : a string identifying the tag in a unique way
+    * `description` : a general description of the tag
+    * `timeType` :  the time type of the tag [time, run-lumi,...]
+    * `payloadSpec` : the type of the object that the tag is associated with (a CrestContainer can generate either `crest-json-single-iov`, or `crest-json-multi-iov`)
+    * `synchronization` : the synchronization of the tag, this is still to be defined, but could be `UPDX` for example (optional, can be taken from tag name).
+    * `lastValidatedTime`:  The last validated time of the tag, in millisecon (optional, not used in Athena).
+    * `endOfValidity` : the end of validity of the tag, in the same units as the "since" of the IOVs (see later).
+1. `TAGMETA` : this object has been introduced mainly to provide a correct mapping with the existing `metadata` of `COOL`, in particular related to `channels` list and `folder` specifications. It is in one-to-one relationship with the `TAG`. It is essential to have these metadata in order to increase compatibility with COOL related code in Athena. We will provide more details below on its content. For Athena usage, it is mandatory to create a TAGMETA entry for each TAG that a user create.
+
+2. `IOV` : the metadata represeting the start of validity for a given payload. It is defined via the following properties
+    * `tagName` : the name of the tag.
+    * `since` : the since of the IOV.
+    * `insertionTime` : the insertion time of the IOV. Provided by the server.
+    * `payloadHash` : the unique key for the payload. It is computed as the SHA256 hash of the payload by the server.
+      In CREST the concept of IOV is always *open-ended*, in the sense that it is valid until the next since. The only IOV for which we foresee a possible *end time* is the last IOV; when the *end time* is not INFINITY, then it can be provided during the PAYLOAD upload. It will update the *endOfValidity* field in the TAG table. Any search functionality will in any case only use the *since* field in CREST, and eventually the *insertionTime*. The latter is used to *go back* into the history of a given *since*, in the case that it has been overridden by the expert with another PAYLOAD.
+3. `PAYLOAD` : it is the binary object containing the conditions data payload. In our default serialization this is a JSON file, created with the help of `CrestContainer`. This allows to transparently migrate existing COOL data in a format that can then be used to generate a generic `CoralAttributeList` container on the Athena side. In terms of server implementation, the CREST server itself and its underlying relational DB tables are completely agnostic as far as the serialization choice of the PAYLOAD is concerned. For obvious reason, that is not at all the case on the client side, so it is important to understand the implication of the chosen serialization, and eventually adopt a strategy which is optimal for the given payload type.
+
+The CREST API can then be seen as a *KEY=VALUE* store for PAYLOAD files. The TAG identifies a *payload type* (a combination of *folder* and *tag* in COOL), and every TAG will contain the *time history* of the PAYLOAD. When looking at the IOV in a tag, we get a unique *KEY* (the `payloadHash`) corresponding to a given time. Using that *KEY* we can download the corresponding PAYLOAD in a reproducible way, satisfying the basic principle of a REST interface.
+
+## Workflows
+The typical operational workflows can be described in the following way.
+An expert having a specific payload to upload over time (a sort of conditions type) should proceed with the following steps in order to upload to CREST.
+1. Create a `TAG` if it does not exists.
+2. Upload `IOV` and `PAYLOAD` to an existing `TAG`.
+3. Find an existing `TAG`.
+4. Load the `IOV` for a given `TAG`.
+5. Create a `GLOBALTAG`.
+6. Associate a given `TAG` to one or more `GLOBALTAG`s.
+7. Find a `GLOBALTAG` and retrieve the associated `TAG`s via the `GLOBALTAGMAP`.

Dockerfile

@@ -1,45 +1,42 @@
 # CrestDB
-#
-# VERSION       CrestDB-1.0
-
-# use the centos base image provided by dotCloud
-# FROM openjdk:8u121-jdk
-# FROM anapsix/alpine-java
-# FROM openjdk:8u212-jre-alpine3.9
-#FROM openjdk:15-jdk-alpine
-FROM adoptopenjdk/openjdk11:alpine-jre
-MAINTAINER Andrea Formica
-
-ENV USR crest
-ENV CREST_GID 208
-
-ENV crest_version 1.0-SNAPSHOT
-ENV crest_dir /home/${USR}/crest
-ENV data_dir /home/${USR}/data
-#ENV data_dir /data
-ENV gradle_version 6.7
-ENV TZ GMT
-
-RUN addgroup -g $CREST_GID $USR \
-    && adduser -S -u $CREST_GID -G $USR -h /home/$USR $USR
+FROM registry.cern.ch/docker.io/eclipse-temurin:23-alpine
+LABEL maintainer="Andrea Formica"
+
+ENV USR=crestsvc
+ENV CREST_GID=208
+
+ENV crest_version=1.0-SNAPSHOT
+ENV crest_dir=/home/${USR}/crest
+ENV data_dir=/home/${USR}/data
+ENV config_dir=/home/${USR}/config
+ENV TZ=GMT
+
+## RUN groupadd -g 208 crest && adduser -u $CREST_GID -g $CREST_GID -d /home/${USR} ${USR} && usermod -aG crest ${USR}
+RUN addgroup -g $CREST_GID crest \
+    && adduser -u $CREST_GID -G crest -h /home/${USR} -D ${USR} \
+    && addgroup ${USR} crest
 
 RUN  mkdir -p ${crest_dir} \
+  && mkdir -p ${config_dir} \
   && mkdir -p ${data_dir}/web \
   && mkdir -p ${data_dir}/dump \
-  && mkdir -p ${data_dir}/logs
+  && mkdir -p ${data_dir}/logs \
+  && chown -R ${CREST_GID}:${CREST_GID} /home/${USR}
 
 ## This works if using an externally generated war, in the local directory
-ADD crestdb-web/build/libs/crest.war ${crest_dir}/crest.war
-ADD web ${data_dir}/web
-ADD logback.xml.crest ${data_dir}/logback.xml
+ADD build/libs/crest.jar ${crest_dir}/crest.jar
+## ADD web ${data_dir}/web
 
 ### we export only 1 directories....
 VOLUME "${data_dir}"
 EXPOSE 8080
 
 # copy the entrypoint
 COPY ./entrypoint.sh /home/${USR}
-COPY ./create-properties.sh /home/${USR}
+COPY ./logback.xml.crest /home/${USR}/logback.xml
+## This is not needed in swarm deployment, only for local testing.
+COPY ./javaopts.properties /home/${USR}
+#COPY ./create-properties.sh /home/${USR}
 
 RUN chown -R $USR:$CREST_GID /home/${USR}
 

Makefile

@@ -7,30 +7,30 @@ else
     SPECFILE        = $(shell find . -maxdepth 1 -type f -name *.spec)
 endif
 
-SPECFILE_NAME       = $(shell awk '$$1 == "Name:"     { print $$2 }' $(SPECFILE) )
-SPECFILE_VERSION    = $(shell awk '$$1 == "Version:"  { print $$2 }' $(SPECFILE) )
-SPECFILE_RELEASE    = $(shell awk '$$1 == "Release:"  { print $$2 }' $(SPECFILE) )
-TARFILE             = $(SPECFILE_NAME)-$(SPECFILE_VERSION).tgz
+#SPECFILE_NAME       = $(shell awk '$$1 == "Name:"     { print $$2 }' $(SPECFILE) )
+#SPECFILE_VERSION    = $(shell awk '$$1 == "Version:"  { print $$2 }' $(SPECFILE) )
+#SPECFILE_RELEASE    = $(shell awk '$$1 == "Release:"  { print $$2 }' $(SPECFILE) )
+#TARFILE             = $(SPECFILE_NAME)-$(SPECFILE_VERSION).tgz
 DIST                = $(shell rpm --eval %{dist})
-CREST_VERSION       = 2.0
-CREST_RELEASE       = $(shell sed -nr '/release=/ s/.*release=([^"]+).*/\1/p' $(PWD)/crestdb-web/src/main/resources/messages.properties)
+CREST_VERSION       = $(shell sed -nr '/version=/ s/.*version=([^"]+).*/\1/p' $(PWD)/src/main/resources/rpm.properties)
+CREST_RELEASE       = $(shell sed -nr '/release=/ s/.*release=([^"]+).*/\1/p' $(PWD)/src/main/resources/rpm.properties)
+$(info CREST version : $(CREST_VERSION))
+$(info CREST release : $(CREST_RELEASE))
+
 TARGET_DIR          = "crest-dist"
-CREST_TARFILE       = $(SPECFILE_NAME)-$(CREST_VERSION).tgz
-CREST_WAR           = $(shell find ./crestdb-web/ -maxdepth 3 -type f -name "crest.war")
-CREST_IMAGE         = "gitlab-registry.cern.ch/formica/crest:$(CREST_VERSION)"
-##CREST_IMAGE         = "crest-test"
+CREST_JAR           = $(shell find ./ -maxdepth 3 -type f -name "crest.jar")
 ## Commands
 MD = mkdir
 CP = cp
 DOCK = docker
 GRADLE = ./gradlew
-sources:
-    ifeq ($(UNAME_S), Darwin)
-        $(info create tar for MACOSX)
-	    tar -zcvf --exclude='.git' --exclude='.gitignore' --transform 's,^,$(SPECFILE_NAME)-$(SPECFILE_VERSION)/,' $(TARFILE) src/*
-    else
-	    tar -zcvf $(TARFILE) --exclude-vcs --transform 's,^,$(SPECFILE_NAME)-$(SPECFILE_VERSION)/,' src/*
-    endif
+#sources:
+#    ifeq ($(UNAME_S), Darwin)
+#        $(info create tar for MACOSX)
+#	    tar -zcvf --exclude='.git' --exclude='.gitignore' --transform 's,^,$(SPECFILE_NAME)-$(SPECFILE_VERSION)/,' $(TARFILE) src/*
+#    else
+#	    tar -zcvf $(TARFILE) --exclude-vcs --transform 's,^,$(SPECFILE_NAME)-$(SPECFILE_VERSION)/,' src/*
+#    endif
 
 clean:
 	rm -rf build/ $(TARFILE)
@@ -40,7 +40,7 @@ rpm: sources
 srpm: sources
 	rpmbuild -bs --define 'dist $(DIST)' --define "_topdir $(PWD)/build" --define '_sourcedir $(PWD)' $(SPECFILE)
 mrpm:
-	rpmbuild -bb --define "_topdir $(PWD)/build" --define "_sourcedir $(PWD)" --define "version $(COOLR_VERSION)" --define "release $(COOLR_RELEASE)" $(SPECFILE)
+	rpmbuild -bb --define "_topdir $(PWD)/build" --define "_sourcedir $(PWD)" --define "version $(CREST_VERSION)" --define "release $(CREST_RELEASE)" $(SPECFILE)
 
 dist:
 	rm -rf $(TARGET_DIR)
@@ -49,7 +49,7 @@ dist:
 build: clean
 	$(GRADLE) clean build
 package: dist
-	$(CP) $(CREST_WAR) $(TARGET_DIR)/crest.war
+	$(CP) $(CREST_JAR) $(TARGET_DIR)/crest.jar
 	$(CP) ./logback.xml.crest $(TARGET_DIR)/logback.xml
 	$(CP) ./javaopts.properties.rpm $(TARGET_DIR)/javaopts.properties
 	$(CP) ./entrypoint.sh $(TARGET_DIR)/entrypoint.sh