Create vivo/home on application start-up (vivo-project#192)

* Create vivo/home on application start-up - Add logic for untarring vivo-home Related to: https://jira.lyrasis.org/browse/VIVO-1443 * Require runtime.properties and applicationSetup.n3 be in 'config/' dir Related to: https://jira.lyrasis.org/browse/VIVO-1443 * Support loading config files with or without 'default' prefix * Prioritize without the prefix Related to: https://jira.lyrasis.org/browse/VIVO-1443 * Add leading slash to location of vivo-home.tar resource path * This allows for deployment in both Tomcat and Jetty Related to: https://jira.lyrasis.org/browse/VIVO-1443 * Ensure VIVO_HOME is populated whether it is empty or not Related to: https://jira.lyrasis.org/browse/VIVO-1443 * Require common properties to be in JNDI Properties include: - vitro/home - vitro/appName - vitro/rootUserAddress - vitro/defaultNamespace Related to: https://jira.lyrasis.org/browse/VIVO-1443 * VIVO-1443: non destructive vivo home untarring (vivo-project#2) * Upgrade Jena version to 3.16.0 (vivo-project#196) Related to: https://jira.lyrasis.org/browse/VIVO-1943 * remove example-settings.xml * update home directory untar non-destructive * checksum digest to retain modified files * compare checksum from digest with existing file * overwrite files that have not changed * add command to manually generate checksum digest * simplify parsing checksum digest using pattern Co-authored-by: Andrew Woods <[email protected]> Co-authored-by: Andrew Woods <[email protected]> * not overwrite existing file if same as tar entry (vivo-project#3) * Upgrade Jena version to 3.16.0 (vivo-project#196) Related to: https://jira.lyrasis.org/browse/VIVO-1943 * remove example-settings.xml * update home directory untar non-destructive * checksum digest to retain modified files * compare checksum from digest with existing file * overwrite files that have not changed * add command to manually generate checksum digest * simplify parsing checksum digest using pattern * do not overwrite file if same as already exists Co-authored-by: Andrew Woods <[email protected]> Co-authored-by: Andrew Woods <[email protected]> * Update log messages for clarity Co-authored-by: Andrew Woods <[email protected]> Co-authored-by: William Welling <[email protected]> Co-authored-by: Gross, Benjamin <[email protected]>
grahamtriggs · Feb 5, 2021 · 8b9a748 · 8b9a748
1 parent 936305b
commit 8b9a748
Show file tree

Hide file tree

Showing 13 changed files with 412 additions and 139 deletions.
diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/application/ApplicationSetup.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/application/ApplicationSetup.java
@@ -24,6 +24,7 @@
  */
 public class ApplicationSetup implements ServletContextListener {
 	private static final String APPLICATION_SETUP_PATH = "config/applicationSetup.n3";
+	private static final String APPLICATION_SETUP_DEFAULT_PATH = "config/default.applicationSetup.n3";
 
 	private ServletContext ctx;
 	private StartupStatus ss;
@@ -45,6 +46,8 @@ public void contextInitialized(ServletContextEvent sce) {
 			this.vitroHomeDir = VitroHomeDirectory.find(ctx);
 			ss.info(this, vitroHomeDir.getDiscoveryMessage());
 
+			this.vitroHomeDir.populate();
+
 			locateApplicationConfigFile();
 			loadApplicationConfigFile();
 			createConfigurationBeanLoader();
@@ -62,11 +65,20 @@ public void contextInitialized(ServletContextEvent sce) {
 
 	private void locateApplicationConfigFile() {
 		Path path = this.vitroHomeDir.getPath().resolve(APPLICATION_SETUP_PATH);
+
+		if (!Files.exists(path) || !Files.isReadable(path)) {
+			path = this.vitroHomeDir.getPath().resolve(APPLICATION_SETUP_DEFAULT_PATH);
+		}
+
 		if (!Files.exists(path)) {
-			throw new IllegalStateException("'" + path + "' does not exist.");
+			throw new IllegalStateException("Neither '" + APPLICATION_SETUP_PATH + "' nor '" +
+					APPLICATION_SETUP_DEFAULT_PATH + "' were found in " +
+					this.vitroHomeDir.getPath());
 		}
 		if (!Files.isReadable(path)) {
-			throw new IllegalStateException("Can't read '" + path + "'");
+			throw new IllegalStateException("No readable '" + APPLICATION_SETUP_PATH + "' nor '" +
+					APPLICATION_SETUP_DEFAULT_PATH + "' files were found in " +
+					this.vitroHomeDir.getPath());
 		}
 		this.configFile = path;
 	}

diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/application/VitroHomeDirectory.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/application/VitroHomeDirectory.java
@@ -4,25 +4,46 @@
 
 import static edu.cornell.mannlib.vitro.webapp.application.BuildProperties.WEBAPP_PATH_BUILD_PROPERTIES;
 
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
-import javax.naming.InitialContext;
 import javax.servlet.ServletContext;
 
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
+import edu.cornell.mannlib.vitro.webapp.config.ContextProperties;
+
 /**
- * Encapsulates some of the info relating to the Vitro home directory.
+ * Encapsulates some of the info relating to and initializes the Vitro home directory.
  */
 public class VitroHomeDirectory {
 	private static final Log log = LogFactory.getLog(VitroHomeDirectory.class);
 
+	private static final String DIGEST_FILE_NAME = "digest.md5";
+
+	private static final Pattern CHECKSUM_PATTERN = Pattern.compile("^[a-f0-9]{32} \\*.+$");
+
 	public static VitroHomeDirectory find(ServletContext ctx) {
 		HomeDirectoryFinder finder = new HomeDirectoryFinder(ctx);
 		return new VitroHomeDirectory(ctx, finder.getPath(),
@@ -52,6 +73,219 @@ public String getDiscoveryMessage() {
 		return discoveryMessage;
 	}
 
+	/**
+	 * Populates VIVO home directory with files required to run.
+	 * 
+	 * NOTE: Will not overwrite any modified files on redeploy.
+	 */
+	public void populate() {
+		File vhdDir = getPath().toFile();
+
+		if (!vhdDir.isDirectory() || vhdDir.list() == null) {
+			throw new RuntimeException("Application home dir is not a directory! " + vhdDir);
+		}
+
+		Map<String, String> digest = untar(vhdDir);
+
+		writeDigest(digest);
+	}
+
+	/**
+	 * A non-destructive untar process that returns checksum digest of tarred files.
+	 * 
+	 * Checksum digest can be manually created with the following command.
+	 * 
+	 * `find /vivo/home -type f | cut -c3- | grep -E '^bin/|^config/|^rdf/' | xargs md5sum > /vivo/home/digest.md5`
+	 * 
+	 * @param destination VIVO home directory
+	 * @return digest of each files checksum
+	 */
+	private Map<String, String> untar(File destination) {
+		log.info("Syncing VIVO home at: " + destination.getPath());
+
+		Map<String, String> digest = new HashMap<>();
+		Map<String, String> storedDigest = loadDigest();
+
+		TarArchiveEntry tarEntry;
+		try (
+			InputStream homeDirTar = getHomeDirTar();
+			TarArchiveInputStream tarInput = new TarArchiveInputStream(homeDirTar);
+		) {
+			while ((tarEntry = tarInput.getNextTarEntry()) != null) {
+
+				// Use the example configurations
+				String outFilename = tarEntry.getName().replace("example.", "");
+				File outFile = new File(destination, outFilename);
+
+				// Is the entry a directory?
+				if (tarEntry.isDirectory()) {
+					if (!outFile.exists()) {
+						outFile.mkdirs();
+					}
+				} else {
+					// Entry is a File
+					boolean write = true;
+
+					// reading bytes into memory to avoid having to unreliably reset stream
+					byte[] bytes = IOUtils.toByteArray(tarInput);
+					String newFileChecksum = checksum(bytes);
+					digest.put(outFilename, newFileChecksum);
+
+					// if file already exists and stored digest contains the file,
+					// check to determine if it has changed
+					if (outFile.exists() && storedDigest.containsKey(outFilename)) {
+						String existingFileChecksum = checksum(outFile);
+						// if file has not changed in home and is not the same as new file, overwrite
+						write = storedDigest.get(outFilename).equals(existingFileChecksum)
+							&& !existingFileChecksum.equals(newFileChecksum);
+					}
+
+					if (write) {
+						outFile.getParentFile().mkdirs();
+						try (
+							InputStream is = new ByteArrayInputStream(bytes);
+							FileOutputStream fos = new FileOutputStream(outFile);
+						) {
+							IOUtils.copy(is, fos);
+							log.info(outFile.getAbsolutePath() + " source has changed and has not been "
+							         + "edited in home, updated file has been copied to home directory.");
+						}
+					} else {
+						log.debug(outFile.getAbsolutePath() + " has been preserved.");
+					}
+				}
+			}
+		} catch (IOException | NoSuchAlgorithmException e) {
+			throw new RuntimeException("Error creating home directory!", e);
+		}
+
+		return digest;
+	}
+
+	/**
+	 * Load checksum digest of VIVO home directory.
+	 * 
+	 * @return checksum digest
+	 */
+	private Map<String, String> loadDigest() {
+		File storedDigest = new File(getPath().toFile(), DIGEST_FILE_NAME);
+		if (storedDigest.exists() && storedDigest.isFile()) {
+			log.info("Reading VIVO home digest: " + storedDigest.getPath());
+			try {
+				return FileUtils
+					.readLines(storedDigest, StandardCharsets.UTF_8)
+					.stream()
+					.filter(CHECKSUM_PATTERN.asPredicate())
+					.map(this::split)
+					.collect(Collectors.toMap(this::checksumFile, this::checksumValue));
+			} catch (IOException e) {
+				throw new RuntimeException("Error reading VIVO home checksum digest!", e);
+			}
+		}
+		log.info("VIVO home digest not found: " + storedDigest.getPath());
+
+		return new HashMap<>();
+	}
+
+	/**
+	 * Write VIVO home checksum digest following md5 format; `<checksum> *<file>`.
+	 * 
+	 * @param digest checksum digest to write
+	 */
+	private void writeDigest(Map<String, String> digest) {
+		File storedDigest = new File(getPath().toFile(), DIGEST_FILE_NAME);
+		try (
+			FileOutputStream fos = new FileOutputStream(storedDigest); 
+			OutputStreamWriter osw = new OutputStreamWriter(fos);
+		) {
+			for (Map.Entry<String, String> entry : digest.entrySet()) {
+				String filename = entry.getKey();
+				String checksum = entry.getValue();
+				osw.write(String.format("%s *%s\n", checksum, filename));
+			}
+		} catch (IOException e) {
+			throw new RuntimeException("Error writing home directory checksum digest!", e);
+		}
+		log.info("VIVO home digest created: " + storedDigest.getPath());
+	}
+
+	/**
+	 * Split checksum.
+	 * 
+	 * @param checksum checksum delimited by space and asterisks `<checksum> *<file>`
+	 * @return split checksum
+	 */
+	private String[] split(String checksum) {
+		return checksum.split("\\s+");
+	}
+
+	/**
+	 * Get value from split checksum.
+	 * 
+	 * @param checksum split checksum
+	 * @return checksum value
+	 */
+	private String checksumValue(String[] checksum) {
+		return checksum[0];
+	}
+
+	/**
+	 * Return file from split checksum.
+	 * 
+	 * @param checksum split checksum
+	 * @return filename
+	 */
+	private String checksumFile(String[] checksum) {
+		return checksum[1].substring(1);
+	}
+
+	/**
+	 * Get md5 checksum from file.
+	 * 
+	 * @param file file
+	 * @return md5 checksum as string
+	 * @throws IOException
+	 * @throws NoSuchAlgorithmException
+	 */
+	private String checksum(File file) throws IOException, NoSuchAlgorithmException {
+		return checksum(FileUtils.readFileToByteArray(file));
+	}
+
+	/**
+	 * Get md5 checksum from bytes.
+	 * 
+	 * @param bytes bytes from file
+	 * @return md5 checksum as string
+	 * @throws NoSuchAlgorithmException
+	 */
+	private String checksum(byte[] bytes) throws NoSuchAlgorithmException {
+		MessageDigest md = MessageDigest.getInstance("MD5");
+		md.update(bytes);
+		// bytes to hex
+		StringBuilder result = new StringBuilder();
+		for (byte b : md.digest()) {
+			result.append(String.format("%02x", b));
+		}
+
+		return result.toString();
+	}
+
+	/**
+	 * Get prepacked VIVO home tar file as input stream.
+	 * 
+	 * @return input stream of VIVO home tar file
+	 */
+	private InputStream getHomeDirTar() {
+		String tarLocation = "/WEB-INF/resources/home-files/vivo-home.tar";
+		InputStream tar = ctx.getResourceAsStream(tarLocation);
+		if (tar == null) {
+			log.error("Application home tar not found in: " + tarLocation);
+			throw new RuntimeException("Application home tar not found in: " + tarLocation);
+		}
+
+		return tar;
+	}
+
 	/**
 	 * Find something that specifies the location of the Vitro home directory.
 	 * Look in the JDNI environment, the system properties, and the
@@ -92,23 +326,12 @@ public Path getPath() {
 		}
 
 		public void getVhdFromJndi() {
-			try {
-				String vhdPath = (String) new InitialContext()
-						.lookup(VHD_JNDI_PATH);
-				if (vhdPath == null) {
-					log.debug("Didn't find a JNDI value at '" + VHD_JNDI_PATH
-							+ "'.");
-				} else {
-					log.debug("'" + VHD_JNDI_PATH + "' as specified by JNDI: "
-							+ vhdPath);
-					String message = String.format(
-							"JNDI environment '%s' was set to '%s'",
-							VHD_JNDI_PATH, vhdPath);
-					foundLocations.add(new Found(Paths.get(vhdPath), message));
-				}
-			} catch (Exception e) {
-				log.debug("JNDI lookup failed. " + e);
-			}
+			String vhdPath = ContextProperties.findJndiProperty(VHD_JNDI_PATH);
+			log.debug("'" + VHD_JNDI_PATH + "' as specified by JNDI: " + vhdPath);
+			String message = String.format(
+					"JNDI environment '%s' was set to '%s'",
+					VHD_JNDI_PATH, vhdPath);
+			foundLocations.add(new Found(Paths.get(vhdPath), message));
 		}
 
 		private void getVhdFromSystemProperties() {

diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/config/ConfigurationPropertiesImpl.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/config/ConfigurationPropertiesImpl.java
@@ -32,8 +32,10 @@ public class ConfigurationPropertiesImpl extends ConfigurationProperties {
 
 	public ConfigurationPropertiesImpl(InputStream stream,
 			Map<String, String> preemptiveProperties,
-			Map<String, String> buildProperties) throws IOException {
+			Map<String, String> buildProperties,
+			Map<String, String> contextProperties) throws IOException {
 		Map<String, String> map = new HashMap<>(buildProperties);
+		map.putAll(contextProperties);
 
 		Properties props = loadFromPropertiesFile(stream);
 		for (String key: props.stringPropertyNames()) {