RFC: PySpark-like, DataFrame-only API

.github/workflows/test.yml

@@ -17,16 +17,18 @@ jobs:
       matrix:
         version:
           - '1.6'
+          - '1.7'
         os:
           - ubuntu-latest
+          - windows-latest
         arch:
           - x64
     steps:
       - uses: actions/checkout@v2
-      - name: Set up JDK 8
+      - name: Set up JDK 11
         uses: actions/setup-java@v2
         with:
-          java-version: '8'
+          java-version: '11'
           distribution: 'adopt'
       - uses: julia-actions/setup-julia@latest
         with:

.gitignore

@@ -2,6 +2,7 @@
 *.jl.mem
 *~
 .idea/
+.vscode/
 target/
 project/
 *.class

Project.toml

@@ -5,15 +5,18 @@ version = "0.5.2"
 [deps]
 IteratorInterfaceExtensions = "82899510-4779-5014-852e-03e436cf321d"
 JavaCall = "494afd89-becb-516b-aafa-70d2670c0337"
+Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
 Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
 Sockets = "6462fe0b-24de-5631-8697-dd941f90decc"
+Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 TableTraits = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c"
+Umlaut = "92992a2b-8ce5-4a9c-bb9d-58be9a7dc841"
 
 [compat]
-JavaCall = "0.7"
-julia = "1"
-TableTraits = "1"
 IteratorInterfaceExtensions = "1"
+JavaCall = "0.7, 0.8"
+TableTraits = "1"
+julia = "1"
 
 [extras]
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"

appveyor.yml

@@ -1,7 +1,7 @@
 environment:
   matrix:
-  - julia_version: 0.7
-  - julia_version: 1
+  - julia_version: 1.6
+  - julia_version: 1.7
   - julia_version: nightly
 
 platform:

jvm/sparkjl/pom.xml

@@ -153,6 +153,11 @@
       <artifactId>scala-library</artifactId>
       <version>${scala.version}</version>
     </dependency>
+    <!-- <dependency>
+      <groupId>org.mdkt.compiler</groupId>
+      <artifactId>InMemoryJavaCompiler</artifactId>
+      <version>1.3.0</version>
+    </dependency> -->
 
     <!-- additional data formats -->
 

src/Spark.jl

@@ -1,50 +1,5 @@
 module Spark
 
-export
-    SparkConf,
-    SparkContext,
-    RDD,
-    JuliaRDD,
-    JavaRDD,
-    text_file,
-    parallelize,
-    map,
-    map_pair,
-    map_partitions,
-    map_partitions_pair,
-    map_partitions_with_index,
-    reduce,
-    filter,
-    collect,
-    count,
-    id,
-    num_partitions,
-    close,
-    @attach,
-    share_variable,
-    @share,
-    flat_map,
-    flat_map_pair,
-    cartesian,
-    group_by_key,
-    reduce_by_key,
-    cache,
-    repartition,
-    coalesce,
-    pipe,
-    # SQL
-    SparkSession,
-    Dataset,
-    sql,
-    read_json,
-    write_json,
-    read_parquet,
-    write_parquet,
-    read_df,
-    write_df
-
-
-
 include("core.jl")
 
-end 
+end

src/chainable.jl

@@ -0,0 +1,63 @@
+"""
+    DotChainer{O, Fn}
+
+See `@chainable` for details.
+"""
+struct DotChainer{O, Fn}
+    obj::O
+    fn::Fn
+end
+
+# DotChainer(obj, fn) = DotChainer{typeof(obj), typeof(fn)}(obj, fn)
+
+(c::DotChainer)(args...) = c.fn(c.obj, args...)
+
+
+"""
+    @chainable T
+
+Adds dot chaining syntax to the type, i.e. automatically translate:
+
+    foo.bar(a)
+
+into
+
+    bar(foo, a)
+
+For single-argument functions also support implicit calls, e.g:
+
+    foo.bar.baz(a, b)
+
+is treated the same as:
+
+    foo.bar().baz(a, b)
+
+Note that `@chainable` works by overloading `Base.getproperty()`,
+making it impossible to customize it for `T`. To have more control,
+one may use the underlying wrapper type - `DotCaller`.
+"""
+macro chainable(T)
+    return quote
+        function Base.getproperty(obj::$(esc(T)), prop::Symbol)
+            if hasfield(typeof(obj), prop)
+                return getfield(obj, prop)
+            elseif isdefined(@__MODULE__, prop)
+                fn = getfield(@__MODULE__, prop)
+                return DotChainer(obj, fn)
+            else
+                error("type $(typeof(obj)) has no field $prop")
+            end
+        end
+    end
+end
+
+
+function Base.getproperty(dc::DotChainer, prop::Symbol)
+    if hasfield(typeof(dc), prop)
+        return getfield(dc, prop)
+    else
+        # implicitely call function without arguments
+        # and propagate getproperty to the returned object
+        return getproperty(dc(), prop)
+    end
+end