mattf/spark repository overview

docker run -it --gpus all mattf/spark:alpha-... spark-shell --master spark://localhost:7077

scala> val df = spark.range(1, 100).select($"id" as "a").join(spark.range(1, 100).select($"id" as "b"), $"a" === $"b")
df: org.apache.spark.sql.DataFrame = [a: bigint, b: bigint]

scala> df.explain
== Physical Plan ==
GpuColumnarToRow false
+- GpuBroadcastHashJoin [a#2L], [b#6L], Inner, GpuBuildRight
   :- GpuProject [id#0L AS a#2L]
   :  +- GpuRange (1, 100, step=1, splits=64)
   +- GpuBroadcastExchange HashedRelationBroadcastMode(List(input[0, bigint, false])), [id=#25]
      +- GpuProject [id#4L AS b#6L]
         +- GpuRange (1, 100, step=1, splits=64)

scala> df.show
+---+---+                                                                       
|  a|  b|
+---+---+
|  1|  1|
|  2|  2|
|  3|  3|
...
only showing top 20 rows

docker run -d -p 8080:8080 -p 7077:7077 mattf/spark:alpha-... start-master

docker run -d --gpus all mattf/spark:alpha-... start-worker spark://node0:7077

docker run -it --net host --entrypoint spark-shell mattf/spark:alpha-... --master spark://node0:7077

open http://node0:8080

kubectl create rolebinding default --clusterrole=edit --serviceaccount=default:default

kubectl create service clusterip mydriver --clusterip=None
kubectl set selector service/mydriver run=mydriver

kubectl run -it mydriver --image=mattf/spark:alpha-... --command -- spark-shell --master k8s://kubernetes