[{"_1":2,"_152":-5,"_153":-5},"loaderData",{"_3":4,"_13":14,"_21":22,"_131":132},"root",{"_5":-5,"_6":-7,"_7":8,"_9":-7,"_10":11,"_12":-7},"currentUser","primaryEmail","siteVerification","6LeIxAcTAAAAAJcZVRqyHh71UMIEGNQ_MXjiZKhI","toast","enableDev",false,"incidentStatus","routes/_layout",{"_15":-5,"_16":17,"_18":17,"_19":11,"_20":11},"username","isAskGordonEnabled",true,"isUsingNewTemplate","showMinimalLanding","showHoneypotTrap","routes/_layout.r.$namespace.$reponame",{"_23":24,"_25":-5,"_26":11,"_27":11,"_28":29,"_30":31,"_32":-5,"_33":11,"_34":35,"_84":-5,"_85":86,"_124":-5,"_125":11,"_126":17,"_127":11,"_128":-5,"_129":130},"canonicalUrl","https://hub.docker.com/r/omriiluz/spark-jobserver/tags","mcpServer","canSeeDockerfile","canSeeAutobuilds","namespace","omriiluz","reponame","spark-jobserver","cloudBuildSource","isAutobuildsAvailable","repo",{"_36":37,"_38":39,"_41":42,"_43":44,"_45":11,"_46":47,"_48":49,"_50":51,"_53":31,"_28":29,"_54":55,"_56":-5,"_57":58,"_59":60,"_61":62,"_63":-5,"_64":58,"_65":66,"_67":11,"_68":29,"_69":70,"_75":17,"_76":77,"_81":29,"_82":83},"categories",[],"content_types",[40],"image","date_registered","2015-02-05T04:47:45.326247Z","description","","is_private","last_updated","2015-11-14T07:54:18.512649Z","last_modified","2024-10-16T13:48:34.145251Z","media_types",[52],"application/vnd.docker.distribution.manifest.v1+prettyjws","name","pull_count",1699,"repository_type","star_count",0,"status",1,"status_description","active","source","collaborator_count","full_description","[![Build Status](https://travis-ci.org/spark-jobserver/spark-jobserver.svg?branch=master)](https://travis-ci.org/spark-jobserver/spark-jobserver)\n\nspark-jobserver provides a RESTful interface for submitting and managing [Apache Spark](http://spark-project.org) jobs, jars, and job contexts.\nThis repo contains the complete Spark job server project, including unit tests and deploy scripts.\nIt was originally started at [Ooyala](http://www.ooyala.com), but this is now the main development repo.\n\nSee [Troubleshooting Tips](doc/troubleshooting.md).\n\n## Users\n\n(Please add yourself to this list!)\n\n- Ooyala\n- Netflix\n- Avenida.com\n- GumGum\n- Fuse Elements\n\n## Features\n\n- *\"Spark as a Service\"*: Simple REST interface for all aspects of job, context management\n- Support for Spark SQL Contexts/jobs and custom job contexts! See [Contexts](doc/contexts.md).\n- Supports sub-second low-latency jobs via long-running job contexts\n- Start and stop job contexts for RDD sharing and low-latency jobs; change resources on restart\n- Kill running jobs via stop context\n- Separate jar uploading step for faster job startup\n- Asynchronous and synchronous job API. Synchronous API is great for low latency jobs!\n- Works with Standalone Spark as well as Mesos and yarn-client\n- Job and jar info is persisted via a pluggable DAO interface\n- Named RDDs to cache and retrieve RDDs by name, improving RDD sharing and reuse among jobs. \n- Supports Scala 2.10 and 2.11\n\n## Version Information\n\n| Version | Spark Version |\n|-------------|---------------|\n| 0.3.1 | 0.9.1 |\n| 0.4.0 | 1.0.2 |\n| 0.4.1 | 1.1.0 |\n| 0.5.0 | 1.2.0 |\n| 0.5.1 | 1.3.0 |\n\nFor release notes, look in the `notes/` directory. They should also be up on [ls.implicit.ly](http://ls.implicit.ly/spark-jobserver/spark-jobserver).\n\n## Quick start / development mode\n\nYou need to have [SBT](http://www.scala-sbt.org/release/docs/Getting-Started/Setup.html) installed.\n\nTo set the current version, do something like this:\n\n export VER=`sbt version | tail -1 | cut -f2`\n\nFrom SBT shell, simply type \"reStart\". This uses a default configuration file. An optional argument is a\npath to an alternative config file. You can also specify JVM parameters after \"---\". Including all the\noptions looks like this:\n\n reStart /path/to/my.conf --- -Xmx8g\n\nNote that reStart (SBT Revolver) forks the job server in a separate process. If you make a code change, simply\ntype reStart again at the SBT shell prompt, it will compile your changes and restart the jobserver. It enables\nvery fast turnaround cycles.\n\nFor example jobs see the job-server-tests/ project / folder.\n\nWhen you use `reStart`, the log file goes to `job-server/job-server-local.log`. There is also an environment variable\nEXTRA_JAR for adding a jar to the classpath.\n\n### WordCountExample walk-through\n\n#### Package Jar - Send to Server\nFirst, to package the test jar containing the WordCountExample: `sbt job-server-tests/package`.\nThen go ahead and start the job server using the instructions above.\n\nLet's upload the jar:\n\n curl --data-binary @job-server-tests/target/job-server-tests-$VER.jar localhost:8090/jars/test\n OK⏎\n\n#### Ad-hoc Mode - Single, Unrelated Jobs (Transient Context)\nThe above jar is uploaded as app `test`. Next, let's start an ad-hoc word count job, meaning that the job\nserver will create its own SparkContext, and return a job ID for subsequent querying:\n\n curl -d \"input.string = a b c a b see\" 'localhost:8090/jobs?appName=test&classPath=spark.jobserver.WordCountExample'\n {\n \"status\": \"STARTED\",\n \"result\": {\n \"jobId\": \"5453779a-f004-45fc-a11d-a39dae0f9bf4\",\n \"context\": \"b7ea0eb5-spark.jobserver.WordCountExample\"\n }\n }⏎\n\nNOTE: If you want to feed in a text file config and POST using curl, you want the `--data-binary` option, otherwise\ncurl will munge your line separator chars. Like:\n\n curl --data-binary @my-job-config.json 'localhost:8090/jobs?appNam=...'\n\nFrom this point, you could asynchronously query the status and results:\n\n curl localhost:8090/jobs/5453779a-f004-45fc-a11d-a39dae0f9bf4\n {\n \"status\": \"OK\",\n \"result\": {\n \"a\": 2,\n \"b\": 2,\n \"c\": 1,\n \"see\": 1\n }\n }⏎\n\nNote that you could append `&sync=true` when you POST to /jobs to get the results back in one request, but for\nreal clusters and most jobs this may be too slow.\n\n#### Persistent Context Mode - Faster & Required for Related Jobs\nAnother way of running this job is in a pre-created context. Start a new context:\n\n curl -d \"\" 'localhost:8090/contexts/test-context?num-cpu-cores=4&memory-per-node=512m'\n OK⏎\n\nYou can verify that the context has been created:\n\n curl localhost:8090/contexts\n [\"test-context\"]⏎\n\nNow let's run the job in the context and get the results back right away:\n\n curl -d \"input.string = a b c a b see\" 'localhost:8090/jobs?appName=test&classPath=spark.jobserver.WordCountExample&context=test-context&sync=true'\n {\n \"status\": \"OK\",\n \"result\": {\n \"a\": 2,\n \"b\": 2,\n \"c\": 1,\n \"see\": 1\n }\n }⏎\n\nNote the addition of `context=` and `sync=true`.\n\n## Create a Job Server Project\nIn your `build.sbt`, add this to use the job server jar:\n\n\tresolvers += \"Job Server Bintray\" at \"https://dl.bintray.com/spark-jobserver/maven\"\n\n\tlibraryDependencies += \"spark.jobserver\" % \"job-server-api\" % \"0.5.0\" % \"provided\"\n\nFor most use cases it's better to have the dependencies be \"provided\" because you don't want SBT assembly to include the whole job server jar.\n\nTo create a job that can be submitted through the job server, the job must implement the `SparkJob` trait. \nYour job will look like:\n```scala\nobject SampleJob extends SparkJob {\n override def runJob(sc:SparkContext, jobConfig: Config): Any = ???\n override def validate(sc:SparkContext, config: Config): SparkJobValidation = ???\n}\n```\n\n- `runJob` contains the implementation of the Job. The SparkContext is managed by the JobServer and will be provided to the job through this method.\n This releaves the developer from the boiler-plate configuration management that comes with the creation of a Spark job and allows the Job Server to\nmanage and re-use contexts.\n- `validate` allows for an initial validation of the context and any provided configuration. If the context and configuration are OK to run the job, returning `spark.jobserver.SparkJobValid` will let the job execute, otherwise returning `spark.jobserver.SparkJobInvalid(reason)` prevents the job from running and provides means to convey the reason of failure. In this case, the call immediatly returns an `HTTP/1.1 400 Bad Request` status code. \n`validate` helps you preventing running jobs that will eventually fail due to missing or wrong configuration and save both time and resources. \n\nLet's try running our sample job with an invalid configuration:\n\n curl -i -d \"bad.input=abc\" 'localhost:8090/jobs?appName=test&classPath=spark.jobserver.WordCountExample'\n\n HTTP/1.1 400 Bad Request\n Server: spray-can/1.2.0\n Date: Tue, 10 Jun 2014 22:07:18 GMT\n Content-Type: application/json; charset=UTF-8\n Content-Length: 929\n\n {\n \"status\": \"VALIDATION FAILED\",\n \"result\": {\n \"message\": \"No input.string config param\",\n \"errorClass\": \"java.lang.Throwable\",\n \"stack\": [\"spark.jobserver.JobManagerActor$$anonfun$spark$jobserver$JobManagerActor$$getJobFuture$4.apply(JobManagerActor.scala:212)\", \n \"scala.concurrent.impl.Future$PromiseCompletingRunnable.liftedTree1$1(Future.scala:24)\", \n \"scala.concurrent.impl.Future$PromiseCompletingRunnable.run(Future.scala:24)\", \n \"akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:42)\",\n \"akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:386)\", \n \"scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)\", \n \"scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)\", \n \"scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)\", \n \"scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)\"]\n }\n }\n\n### Using Named RDDs\nNamed RDDs are a way to easily share RDDs among job. Using this facility, computed RDDs can be cached with a given name and later on retrieved.\nTo use this feature, the SparkJob needs to mixin `NamedRddSupport`:\n```scala\nobject SampleNamedRDDJob extends SparkJob with NamedRddSupport {\n override def runJob(sc:SparkContext, jobConfig: Config): Any = ???\n override def validate(sc:SparkContext, config: Config): SparkJobValidation = ???\n}\n```\n\nThen in the implementation of the job, RDDs can be stored with a given name:\n```scala\nthis.namedRdds.update(\"french_dictionary\", frenchDictionaryRDD)\n```\nOther job running in the same context can retrieve and use this RDD later on:\n```scala\nval rdd = this.namedRdds.get[(String, String)](\"french_dictionary\").get \n```\n(note the explicit type provided to get. This will allow to cast the retrieved RDD that otherwise is of type RDD[_])\n\nFor jobs that depends on a named RDDs it's a good practice to check for the existence of the NamedRDD in the `validate` method as explained earlier:\n```scala \ndef validate(sc:SparkContext, config: Contig): SparkJobValidation = {\n ...\n val rdd = this.namedRdds.get[(Long, scala.Seq[String])](\"dictionary\")\n if (rdd.isDefined) SparkJobValid else SparkJobInvalid(s\"Missing named RDD [dictionary]\")\n}\n```\n\n## Deployment\n\n1. Copy `config/local.sh.template` to `.sh` and edit as appropriate.\n2. `bin/server_deploy.sh ` -- this packages the job server along with config files and pushes\n it to the remotes you have configured in `.sh`\n3. On the remote server, start it in the deployed directory with `server_start.sh` and stop it with `server_stop.sh`\n\n\nNote: to test out the deploy to a local staging dir, or package the job server for Mesos,\nuse `bin/server_package.sh `.\n\n## Architecture\n\nThe job server is intended to be run as one or more independent processes, separate from the Spark cluster\n(though it very well may be colocated with say the Master).\n\nAt first glance, it seems many of these functions (eg job management) could be integrated into the Spark standalone master. While this is true, we believe there are many significant reasons to keep it separate:\n\n- We want the job server to work for Mesos and YARN as well\n- Spark and Mesos masters are organized around \"applications\" or contexts, but the job server supports running many discrete \"jobs\" inside a single context\n- We want it to support Shark functionality in the future\n- Loose coupling allows for flexible HA arrangements (multiple job servers targeting same standalone master, or possibly multiple Spark clusters per job server)\n\nFlow diagrams are checked in in the doc/ subdirectory. .diagram files are for websequencediagrams.com... check them out, they really will help you understand the flow of messages between actors.\n\n## API\n\n### Jars\n\n GET /jars - lists all the jars and the last upload timestamp\n POST /jars/ - uploads a new jar under \n\n### Contexts\n\n GET /contexts - lists all current contexts\n POST /contexts/ - creates a new context\n DELETE /contexts/ - stops a context and all jobs running in it\n\n### Jobs\n\nJobs submitted to the job server must implement a `SparkJob` trait. It has a main `runJob` method which is\npassed a SparkContext and a typesafe Config object. Results returned by the method are made available through\nthe REST API.\n\n GET /jobs - Lists the last N jobs\n POST /jobs - Starts a new job, use ?sync=true to wait for results\n GET /jobs/ - Gets the result or status of a specific job\n DELETE /jobs/ - Kills the specified job\n GET /jobs//config - Gets the job configuration\n\n### Context configuration\n\nA number of context-specific settings can be controlled when creating a context (POST /contexts) or running an\nad-hoc job (which creates a context on the spot). For example, add urls of dependent jars for a context.\n\n POST '/contexts/my-new-context?dependent-jar-uris=file:///some/path/of/my-foo-lib.jar'\n\nWhen creating a context via POST /contexts, the query params are used to override the default configuration in\nspark.context-settings. For example,\n\n POST /contexts/my-new-context?num-cpu-cores=10\n\nwould override the default spark.context-settings.num-cpu-cores setting.\n\nWhen starting a job, and the context= query param is not specified, then an ad-hoc context is created. Any\nsettings specified in spark.context-settings will override the defaults in the job server config when it is\nstarted up.\n\nAny spark configuration param can be overridden either in POST /contexts query params, or through `spark\n.context-settings` job configuration. In addition, `num-cpu-cores` maps to `spark.cores.max`, and `mem-per-\nnode` maps to `spark.executor.memory`. Therefore the following are all equivalent:\n\n POST /contexts/my-new-context?num-cpu-cores=10\n\n POST /contexts/my-new-context?spark.cores.max=10\n\nor in the job config when using POST /jobs,\n\n spark.context-settings {\n spark.cores.max = 10\n }\n\nTo pass settings directly to the sparkConf that do not use the \"spark.\" prefix \"as-is\", use the \"passthrough\" section.\n\n spark.context-settings {\n spark.cores.max = 10\n passthrough {\n some.custom.hadoop.config = \"192.168.1.1\"\n }\n }\n\nFor the exact context configuration parameters, see JobManagerActor docs as well as application.conf.\n\n### Job Result Serialization\n\nThe result returned by the `SparkJob` `runJob` method is serialized by the job server into JSON for routes\nthat return the result (GET /jobs with sync=true, GET /jobs/). Currently the following types can be\nserialized properly:\n\n- String, Int, Long, Double, Float, Boolean\n- Scala Map's with string key values (non-string keys may be converted to strings)\n- Scala Seq's\n- Array's\n- Anything that implements Product (Option, case classes) -- they will be serialized as lists\n- Maps and Seqs may contain nested values of any of the above\n\nIf we encounter a data type that is not supported, then the entire result will be serialized to a string.\n\n## Contribution and Development\nContributions via Github Pull Request are welcome. See the TODO for some ideas.\n\n- If you need to build with a specific scala version use ++x.xx.x followed by the regular command,\nfor instance: `sbt ++2.11.6 job-server/compile` \n- From the \"master\" project, please run \"test\" to ensure nothing is broken.\n - You may need to set `SPARK_LOCAL_IP` to `localhost` to ensure Akka port can bind successfully\n- Logging for tests goes to \"job-server-test.log\"\n- Run `scoverage:test` to check the code coverage and improve it\n- Please run scalastyle to ensure your code changes don't break the style guide\n- Do \"reStart\" from SBT for quick restarts of the job server process\n- Please update the g8 template if you change the SparkJob API\n\n### Publishing packages\n\n- Be sure you are in the master project\n- Run `+test` to ensure all tests pass for all scala versions\n- Now just run `+publish` and package will be published to bintray\n\nTo announce the release on [ls.implicit.ly](http://ls.implicit.ly/), use\n[Herald](https://github.com/n8han/herald#install) after adding release notes in\nthe `notes/` dir. Also regenerate the catalog with `lsWriteVersion` SBT task\nand `lsync`, in project job-server.\n\nTODO: Automate the above steps with `sbt-release`.\n\n## Contact\n\nFor user/dev questions, we are using google group for discussions:\n\n\nPlease report bugs/problems to:\n\n\n## License\nApache 2.0, see LICENSE.md\n\nCopyright(c) 2014, Ooyala, Inc.\n\n## TODO\n\n- Add Swagger support. See the spray-swagger project.\n- Implement an interactive SQL window. See: [spark-admin](https://github.com/adatao/spark-admin)\n\n- Stream the current job progress via a Listener\n- Add routes to return stage info for a job. Persist it via DAO so that we can always retrieve stage / performance info\n even for historical jobs. This would be pretty kickass.\n","has_starred","hub_user","immutable_tags_settings",{"_71":11,"_72":73},"enabled","rules",[74],".*","is_automated","permissions",{"_78":17,"_79":11,"_80":11},"read","write","admin","user","storage_size",914179029,"orgData","recentTags",[87],{"_88":40,"_89":90,"_91":92,"_93":94,"_95":96,"_97":98,"_46":113,"_114":90,"_115":29,"_116":52,"_53":117,"_118":119,"_120":104,"_121":113,"_122":62,"_123":17},"content_type","creator",146363,"digest","sha256:5e9b6fd3d995f5e2e0508928f3697c43556634b00ab923f0eca95e352754ca98","full_size",914090929,"id",113221,"images",[99],{"_100":101,"_91":92,"_102":44,"_103":104,"_105":106,"_107":44,"_108":44,"_109":-5,"_110":111,"_59":62,"_112":-5},"architecture","amd64","features","last_pulled","2026-09-12T18:35:53.092711846Z","last_pushed","2015-11-14T07:54:17Z","os","os_features","os_version","size",914091537,"variant","2015-11-14T07:54:24.313182Z","last_updater","last_updater_username","media_type","latest","repository",134792,"tag_last_pulled","tag_last_pushed","tag_status","v2","sbxKitMetadata","isPotentialSpam","isHubSbxKitsEnabled","displaySecurityMetadata","tagAttestations","tagsHaveAttestations",[],"routes/_layout.r.$namespace.$reponame.tags",{"_34":133,"_140":141,"_129":150,"_151":11},{"_36":134,"_38":135,"_41":42,"_43":44,"_45":11,"_46":47,"_48":49,"_50":136,"_53":31,"_28":29,"_54":55,"_56":-5,"_57":58,"_59":60,"_61":62,"_63":-5,"_64":58,"_65":66,"_67":11,"_68":29,"_69":137,"_75":17,"_76":139,"_81":29,"_82":83},[],[40],[52],{"_71":11,"_72":138},[74],{"_78":17,"_79":11,"_80":11},"tags",{"_142":60,"_143":-5,"_144":-5,"_145":146},"count","next","previous","results",[147],{"_88":40,"_89":90,"_91":92,"_93":94,"_95":96,"_97":148,"_46":113,"_114":90,"_115":29,"_116":52,"_53":117,"_118":119,"_120":104,"_121":113,"_122":62,"_123":17},[149],{"_100":101,"_91":92,"_102":44,"_103":104,"_105":106,"_107":44,"_108":44,"_109":-5,"_110":111,"_59":62,"_112":-5},[],"canDeleteTags","actionData","errors"]

omriiluz/spark-jobserver

Tags for omriiluz/spark-jobserver