added files for support for mariadb

.github/workflows/tests.yml

@@ -40,5 +40,5 @@ jobs:
       - name: Run tests
         run: |
           source $VENV
-          # Ignore the MySQL test, as it requires a MySQL server:
-          pytest --ignore=test/test_run_experiments/test_run_mysql_experiment.py --ignore=test/test_logtables/test_mysql.py
+          # Ignore the MySQL and MariaDB tests, as these require running servers:
+          pytest --ignore=test/test_run_experiments/test_run_mysql_experiment.py --ignore=test/test_logtables/test_mysql.py --ignore=test/test_run_experiments/test_run_mariadb_experiment.py --ignore=test/test_logtables/test_mariadb.py

py_experimenter/database_connector_mariadb.py

@@ -0,0 +1,7 @@
+from configparser import ConfigParser
+from py_experimenter.database_connector_mysql import DatabaseConnectorMYSQL
+
+class DatabaseConnectorMariaDB(DatabaseConnectorMYSQL):
+
+    def __init__(self, experiment_configuration_file_path: ConfigParser, database_credential_file_path):
+        super().__init__(experiment_configuration_file_path, database_credential_file_path, explicit_transactions = False)

py_experimenter/database_connector_mysql.py

@@ -14,11 +14,12 @@ class DatabaseConnectorMYSQL(DatabaseConnector):
     _write_to_database_separator = "', '"
     _prepared_statement_placeholder = '%s'
 
-    def __init__(self, experiment_configuration_file_path: ConfigParser, database_credential_file_path):
+    def __init__(self, experiment_configuration_file_path: ConfigParser, database_credential_file_path, explicit_transactions: bool = True):
         database_credentials = load_config(database_credential_file_path)
         self.host = database_credentials.get('CREDENTIALS', 'host')
         self.user = database_credentials.get('CREDENTIALS', 'user')
         self.password = database_credentials.get('CREDENTIALS', 'password')
+        self.explit_transactions = explicit_transactions
 
         super().__init__(experiment_configuration_file_path)
 
@@ -85,7 +86,8 @@ def _pull_open_experiment(self) -> Tuple[int, List, List]:
         try:
             connection = self.connect()
             cursor = self.cursor(connection)
-            self._start_transaction(connection, readonly=False)
+            if self.explit_transactions:
+                self._start_transaction(connection, readonly=False)
             experiment_id, description, values = self._execute_queries(connection, cursor)
         except Exception as err:
             connection.rollback()

py_experimenter/experimenter.py

@@ -11,6 +11,7 @@
 from py_experimenter import utils
 from py_experimenter.database_connector_lite import DatabaseConnectorLITE
 from py_experimenter.database_connector_mysql import DatabaseConnectorMYSQL
+from py_experimenter.database_connector_mariadb import DatabaseConnectorMariaDB
 from py_experimenter.exceptions import InvalidConfigError, InvalidValuesInConfiguration, NoExperimentsLeftException
 from py_experimenter.experiment_status import ExperimentStatus
 from py_experimenter.result_processor import ResultProcessor
@@ -68,6 +69,8 @@ def __init__(self,
             self.dbconnector = DatabaseConnectorLITE(self.config)
         elif self.config['PY_EXPERIMENTER']['provider'] == 'mysql':
             self.dbconnector = DatabaseConnectorMYSQL(self.config, database_credential_file_path)
+        elif self.config['PY_EXPERIMENTER']['provider'] == 'mariadb':
+            self.dbconnector = DatabaseConnectorMariaDB(self.config, database_credential_file_path)
         else:
             raise ValueError('The provider indicated in the config file is not supported')
 
@@ -141,7 +144,7 @@ def has_option(self, section_name: str, key: str) -> bool:
     def _is_valid_configuration(_config: configparser, database_credential_file_path: str = None) -> bool:
         """
         Checks whether the given experiment configuration is valid, i.e., it contains all necessary fields, the database provider
-        is either mysql or sqlite, and in case of a mysql database provider, that the database credentials are available.
+        is either mysql, mariadb, or sqlite, and in case of a mysql or mariadb database provider, that the database credentials are available.
 
         :param _config: The experiment configuration.
         :type _config: configparser
@@ -161,11 +164,11 @@ def _is_valid_configuration(_config: configparser, database_credential_file_path
             logging.error('Error in config file: DATABASE section must contain provider, database, and table')
             return False
 
-        if _config['PY_EXPERIMENTER']['provider'] not in ['sqlite', 'mysql']:
-            logging.error('Error in config file: DATABASE provider must be either sqlite or mysql')
+        if _config['PY_EXPERIMENTER']['provider'] not in ['sqlite', 'mysql', 'mariadb']:
+            logging.error('Error in config file: DATABASE provider must be either sqlite, mysql, or mariadb')
             return False
 
-        if _config['PY_EXPERIMENTER']['provider'] == 'mysql':
+        if _config['PY_EXPERIMENTER']['provider'] in ['mysql', 'mariadb']:
             credentials = utils.load_config(database_credential_file_path)
             if not {'host', 'user', 'password'}.issubset(set(credentials.options('CREDENTIALS'))):
                 logging.error(

py_experimenter/result_processor.py

@@ -6,6 +6,7 @@
 from py_experimenter.database_connector import DatabaseConnector
 from py_experimenter.database_connector_lite import DatabaseConnectorLITE
 from py_experimenter.database_connector_mysql import DatabaseConnectorMYSQL
+from py_experimenter.database_connector_mariadb import DatabaseConnectorMariaDB
 from py_experimenter.exceptions import InvalidConfigError, InvalidResultFieldError
 
 result_logger = logging.getLogger('result_logger')
@@ -37,6 +38,8 @@ def __init__(self, _config: dict, credential_path, table_name: str, result_field
             self._dbconnector: DatabaseConnector = DatabaseConnectorLITE(_config)
         elif _config['PY_EXPERIMENTER']['provider'] == 'mysql':
             self._dbconnector: DatabaseConnector = DatabaseConnectorMYSQL(_config, credential_path)
+        elif _config['PY_EXPERIMENTER']['provider'] == 'mariadb':
+            self._dbconnector: DatabaseConnector = DatabaseConnectorMariaDB(_config, credential_path)
         else:
             raise InvalidConfigError("Invalid database provider!")
 

test/test_database_connector_mariadb.py

@@ -0,0 +1,31 @@
+from typing import Dict
+
+import pytest
+from py_experimenter.database_connector_mariadb import DatabaseConnectorMariaDB
+
+
+
+@pytest.mark.parametrize(
+    'values, condition, expected',
+    [
+        pytest.param(
+            {'some_key': 'some_value',
+             'some_other_key': 'some_other_value'},
+            'some_condition',
+            'UPDATE some_table SET some_key = %s, some_other_key = %s WHERE some_condition',
+            id='2_values'
+        ),
+        pytest.param(
+            {'some_key': 'some_value'},
+            'some_condition',
+            'UPDATE some_table SET some_key = %s WHERE some_condition',
+            id='1_value'
+        )
+    ]
+)
+def test_prepare_update_query(values: Dict, expected: str, condition: str):
+    class A():
+        _prepared_statement_placeholder = '%s'
+
+    self = A()
+    assert DatabaseConnectorMariaDB._prepare_update_query(self, 'some_table', values,  condition) == expected

test/test_logtables/mariadb_logtables.cfg

@@ -0,0 +1,14 @@
+[PY_EXPERIMENTER]
+provider=mariadb
+database=py_experimenter
+table=test_mariadb_logtables
+n_jobs = 5
+
+keyfields = value:int, exponent:int
+resultfields = sin, cos
+logtables = test_mariadb_log:Log, test_mariadb_log2:Log2
+Log = test:int
+Log2 = test:int
+
+value=1,2,3,4,5,6,7,8,9,10
+exponent=1,2,3

test/test_logtables/test_mariadb.py

@@ -0,0 +1,113 @@
+import os
+from configparser import ConfigParser
+from math import cos, sin
+
+from freezegun import freeze_time
+from mock import call, patch
+
+from py_experimenter.database_connector import DatabaseConnector
+from py_experimenter.database_connector_mariadb import DatabaseConnectorMariaDB
+from py_experimenter.experimenter import PyExperimenter
+from py_experimenter.result_processor import ResultProcessor
+
+
+@patch('py_experimenter.experimenter.DatabaseConnectorMYSQL._create_database_if_not_existing')
+@patch('py_experimenter.experimenter.DatabaseConnectorMYSQL._test_connection')
+@patch('py_experimenter.experimenter.DatabaseConnectorMYSQL.fill_table')
+@patch('py_experimenter.experimenter.DatabaseConnectorMYSQL.connect')
+@patch('py_experimenter.experimenter.DatabaseConnectorMYSQL.cursor')
+@patch('py_experimenter.experimenter.DatabaseConnectorMYSQL.fetchall')
+@patch('py_experimenter.experimenter.DatabaseConnectorMYSQL.close_connection')
+@patch('py_experimenter.experimenter.DatabaseConnectorMYSQL.execute')
+def test_tables_created(execute_mock, close_connection_mock, fetchall_mock, cursor_mock, connect_mock, fill_table_mock, test_connection_mock, create_database_mock):
+    execute_mock.return_value = None
+    fetchall_mock.return_value = None
+    close_connection_mock.return_value = None
+    cursor_mock.return_value = None
+    connect_mock.return_value = None
+    fill_table_mock.return_value = None
+    create_database_mock.return_value = None
+    test_connection_mock.return_value = None
+    experimenter = PyExperimenter(os.path.join('test', 'test_logtables', 'mariadb_logtables.cfg'))
+    experimenter.fill_table_from_config()
+    assert execute_mock.mock_calls[1][1][1] == ('CREATE TABLE test_mariadb_logtables (ID INTEGER PRIMARY KEY AUTO_INCREMENT, value int DEFAULT NULL,'
+                                                'exponent int DEFAULT NULL,creation_date DATETIME DEFAULT NULL,status VARCHAR(255) DEFAULT NULL,'
+                                                'start_date DATETIME DEFAULT NULL,name LONGTEXT DEFAULT NULL,machine VARCHAR(255) DEFAULT NULL,'
+                                                'sin VARCHAR(255) DEFAULT NULL,cos VARCHAR(255) DEFAULT NULL,end_date DATETIME DEFAULT NULL,'
+                                                'error LONGTEXT DEFAULT NULL);')
+    print(execute_mock.mock_calls[2][1][1])
+    assert execute_mock.mock_calls[2][1][1] == ('CREATE TABLE test_mariadb_logtables__test_mariadb_log (ID INTEGER PRIMARY KEY AUTO_INCREMENT,'
+                                                ' experiment_id INTEGER, timestamp DATETIME, test int DEFAULT NULL, FOREIGN KEY (experiment_id)'
+                                                ' REFERENCES test_mariadb_logtables(ID) ON DELETE CASCADE);')
+
+
+@freeze_time("2012-01-14 03:21:34")
+@patch('py_experimenter.result_processor.DatabaseConnectorMariaDB')
+def test_logtable_insertion(database_connector_mock):
+    fixed_time = '2012-01-14 03:21:34'
+    config = ConfigParser()
+    config.read(os.path.join('test', 'test_logtables', 'mariadb_logtables.cfg'))
+    result_processor = ResultProcessor(config, None, None, None, 0)
+    result_processor._table_name = 'some_table_name'
+    result_processor.process_logs({'test_table_0': {'test0': 'test', 'test1': 'test'},
+                                   'test_table_1': {'test0': 'test'}})
+    result_processor._dbconnector.execute_queries.assert_called()
+    result_processor._dbconnector.execute_queries.assert_called_with(
+        [f'INSERT INTO some_table_name__test_table_0 (test0, test1, experiment_id, timestamp) VALUES (test, test, 0, \'{fixed_time}\')',
+         f'INSERT INTO some_table_name__test_table_1 (test0, experiment_id, timestamp) VALUES (test, 0, \'{fixed_time}\')'])
+
+
+@patch('py_experimenter.experimenter.DatabaseConnectorMYSQL._create_database_if_not_existing')
+@patch('py_experimenter.experimenter.DatabaseConnectorMYSQL._test_connection')
+@patch('py_experimenter.experimenter.DatabaseConnectorMYSQL.connect')
+@patch('py_experimenter.experimenter.DatabaseConnectorMYSQL.cursor')
+@patch('py_experimenter.experimenter.DatabaseConnectorMYSQL.commit')
+@patch('py_experimenter.experimenter.DatabaseConnectorMYSQL.fetchall')
+@patch('py_experimenter.experimenter.DatabaseConnectorMYSQL.close_connection')
+@patch('py_experimenter.experimenter.DatabaseConnectorMYSQL.execute')
+def test_delete_logtable(execution_mock, close_connection_mock, commit_mocck, fetchall_mock, cursor_mock, connect_mock, test_connection_mock, create_database_mock):
+    fetchall_mock.return_value = cursor_mock.return_value = connect_mock.return_value = commit_mocck.return_value = None
+    close_connection_mock.return_value = test_connection_mock.return_value = create_database_mock.return_value = execution_mock.return_value = None
+    experimenter = PyExperimenter(os.path.join('test', 'test_logtables', 'mariadb_logtables.cfg'))
+    experimenter.delete_table()
+    execution_mock.assert_has_calls([call(None, 'DROP TABLE IF EXISTS test_mariadb_logtables__test_mariadb_log'),
+                                     call(None, 'DROP TABLE IF EXISTS test_mariadb_logtables__test_mariadb_log2'),
+                                     call(None, 'DROP TABLE IF EXISTS test_mariadb_logtables')])
+
+
+# Integration Test
+def own_function(keyfields: dict, result_processor: ResultProcessor, custom_fields: dict):
+    # run the experiment with the given value for the sin and cos function
+    sin_result = sin(keyfields['value'])**keyfields['exponent']
+    cos_result = cos(keyfields['value'])**keyfields['exponent']
+
+    # write result in dict with the resultfield as key
+    result = {'sin': sin_result, 'cos': cos_result}
+
+    # send result to to the database
+    result_processor.process_results(result)
+    result_processor.process_logs({'test_mariadb_log': {'test': 0}, 'test_mariadb_log2': {'test': 1}})
+    result_processor.process_logs({'test_mariadb_log': {'test': 2}, 'test_mariadb_log2': {'test': 3}})
+
+
+def test_integration():
+    experimenter = PyExperimenter(os.path.join('test', 'test_logtables', 'mariadb_logtables.cfg'))
+    try:
+        experimenter.delete_table()
+    except Exception:
+        pass
+    experimenter.fill_table_from_config()
+    experimenter.execute(own_function, 1)
+    connection = experimenter.dbconnector.connect()
+    cursor = experimenter.dbconnector.cursor(connection)
+    cursor.execute(f"SELECT * FROM test_mariadb_logtables__test_mariadb_log")
+    logtable = cursor.fetchall()
+    timesteps = [x[2] for x in logtable]
+    non_timesteps = [x[:2] + x[3:] for x in logtable]
+    assert non_timesteps == [(1, 1, 0), (2, 1, 2)]
+    cursor.execute(f"SELECT * FROM test_mariadb_logtables__test_mariadb_log2")
+    logtable2 = cursor.fetchall()
+    timesteps2 = [x[2] for x in logtable2]
+    logtable2 = [x[:2] + x[3:] for x in logtable2]
+    assert logtable2 == [(1, 1, 1), (2, 1, 3)]
+    assert timesteps == timesteps2

test/test_run_experiments/test_run_mariadb_error_config.cfg

@@ -0,0 +1,9 @@
+[PY_EXPERIMENTER]
+provider=mariadb
+database=py_experimenter
+table=test_mariadb
+n_jobs = 5
+
+keyfields = error_code:NUMERIC
+resultfields = 
+error_code = 0,1,2