Submit a apache spark job from Emacs. I use this with Spacemacs to run spark jobs locally

spark-submit.el

;; Change these
(setq project-name "project-name"
      main-class-name "canonical.name.of.main.class"
      version "0.0.0"
      project-root (projectile-project-root)
      log4j-configuration (s-lex-format "file://${project-root}/log4j.properties")
      config-file (concat project-root "app.conf"))

(defun spark-submit-command(class-name java-options jar-location)
  (let ((command (s-lex-format
                  "spark-submit --class ${class-name} --master local[4] --driver-java-options=\"${java-options}\" \"${jar-location}\"")))
    (message command)
    command))

(defun spark-submit-shell-process(command)
  (start-process-shell-command "spark-submit" "*spark-submit*" command))

(defun my/spark-submit ()
  (interactive)
  (let* ((java-options (s-lex-format
                        "-Dconfig.file=${config-file} -Dextra.config=test -Dlog4j.configuration=${log4j-configuration}"))
         (jar-location (s-lex-format
                        "target/scala-2.11/${project-name}-assembly-${version}.jar"))
         (full-path-to-jar (concat project-root jar-location))
         (command (spark-submit-command main-class-name java-options full-path-to-jar)))
    (pop-to-buffer "*spark-submit*")
    (erase-buffer)
    (insert command)
    (newline 2)
    (spark-submit-shell-process command)))

(defun my/ensime-sbt-do-assembly ()
  "Compile all sources including tests."
  (interactive)
  (sbt:command "assembly"))

(defun my/ensime-sbt-do-scalafmt ()
  "Compile all sources including tests."
  (interactive)
  (sbt:command ";scalafmt; test:scalafmt"))

(defun my/ensime-sbt-do-reload ()
  "Compile all sources including tests."
  (interactive)
  (sbt:command "reload"))

;; Key maps
(spacemacs/declare-prefix "ob" "build")
(spacemacs/declare-prefix "op" "projectile")
(spacemacs/set-leader-keys
  ; build
  "obb" 'my/ensime-sbt-do-assembly
  "obf" 'my/ensime-sbt-do-scalafmt
  "obr" 'my/ensime-sbt-do-reload
  ; projectile
  "ops" 'my/spark-submit)

(message "loaded, good to go")