Update README, CHANGELOG, tests and info

CHANGELOG.md

@@ -1,8 +1,16 @@
 ### Spark-submit
 
+##### 1.1.0 (2021-11-13)
+- Use of `concluded` attribute to keep track of submitted job
+- Addition of `get_submit_cmd()` method
+- Improve `system_info()` functionality
+- More generic status polling
+- Various improvements & bug fixes
+
 ##### 1.0.1 (2021-10-22)
-- Fix wrong return code bug in `get_code()`
+- Fix return code bug in `get_code()`
 - Change argument order in `submit()`
+- Minor improvements
 
 ##### 1.0.0 (2021-10-16)
-- First release
+- First release

README.md

@@ -55,15 +55,17 @@ spark_args = {
     }
 
 app = SparkJob('s3a://bucket/path/some_file.jar', **spark_args)
-print(app.submit_cmd)
+print(app.get_submic_cmd(multiline=True))
 print(app.env_vars)
 
 # monitor state in the background every x seconds with `await_result=x`
 app.submit(await_result=10, use_env_vars=True)
 
 print(app.get_state()) # 'SUBMITTED'
 
-# do other stuff...
+while not app.concluded:
+    # do other stuff...
+    print(app.get_state()) # 'RUNNING'
 
 print(app.get_state()) # 'FINISHED'
 ```
@@ -145,7 +147,7 @@ Note any additional requirements for running the tests: `pip install -r tests/re
 
 #### Additional methods
 
-`spark_submit.system_info()`: Collects Spark related system information, such as versions of spark-submit, Scala, Java, Python and OS
+`spark_submit.system_info()`: Collects Spark related system information, such as versions of spark-submit, Scala, Java, PySpark, Python and OS
 
 `spark_submit.SparkJob.kill()`: Kills the running Spark job (cluster mode only)
 

setup.py

@@ -1,4 +1,4 @@
-"""spark-submit module Installation script"""
+"""spark-submit module installation script"""
 from setuptools import setup, find_packages
 import os
 

spark_submit/__info__.py

@@ -11,7 +11,7 @@
 __author_email__ = '[email protected]'
 __maintainer__ = 'PApostol'
 __license__ = 'MIT'
-__version__ = '1.0.1'
+__version__ = '1.1.0'
 __description__ = 'Python manager for spark-submit jobs'
 __url__ = f'https://github.com/{__author__}/{__title__}'
 __bugtrack_url__ = f'{__url__}/issues'

tests/run_test.py

@@ -3,9 +3,9 @@
 print('---Running example on local mode---')
 job = SparkJob('tests/resources/pyspark_example.py', main_file_args ='100000')
 
-print('spark-submit command:\n' + job.submit_cmd)
-job.submit()
+print('spark-submit command:\n' + job.get_submit_cmd(multiline=True))
 print('\nJob running...\n')
+job.submit()
 
 print('Job output:\n{0}\n'.format(job.get_output()))
 print('Job state: {0}\n'.format(job.get_state()))