[{"_1":2,"_151":-5,"_152":-5},"loaderData",{"_3":4,"_13":14,"_20":21,"_130":131},"root",{"_5":-5,"_6":-7,"_7":8,"_9":-7,"_10":11,"_12":-7},"currentUser","primaryEmail","siteVerification","6LeIxAcTAAAAAJcZVRqyHh71UMIEGNQ_MXjiZKhI","toast","enableDev",false,"incidentStatus","routes/_layout",{"_15":-5,"_16":17,"_18":17,"_19":11},"username","isAskGordonEnabled",true,"isUsingNewTemplate","showMinimalLanding","routes/_layout.r.$namespace.$reponame",{"_22":23,"_24":-5,"_25":11,"_26":11,"_27":28,"_29":30,"_31":-5,"_32":11,"_33":34,"_83":-5,"_84":85,"_123":-5,"_124":11,"_125":17,"_126":11,"_127":-5,"_128":129},"canonicalUrl","https://hub.docker.com/r/mohsenasm/spark-tpc-ds-performance-test-in-docker/tags","mcpServer","canSeeDockerfile","canSeeAutobuilds","namespace","mohsenasm","reponame","spark-tpc-ds-performance-test-in-docker","cloudBuildSource","isAutobuildsAvailable","repo",{"_35":36,"_37":38,"_40":41,"_42":43,"_44":11,"_45":46,"_47":48,"_49":50,"_52":30,"_27":28,"_53":54,"_55":39,"_56":57,"_58":59,"_60":61,"_62":-5,"_63":57,"_64":65,"_66":11,"_67":28,"_68":69,"_74":11,"_75":76,"_80":28,"_81":82},"categories",[],"content_types",[39],"image","date_registered","2019-07-05T13:04:31.150476Z","description","","is_private","last_updated","2019-07-13T20:48:56.117274Z","last_modified","2024-10-16T13:48:34.145251Z","media_types",[51],"application/vnd.docker.container.image.v1+json","name","pull_count",292,"repository_type","star_count",0,"status",1,"status_description","active","source","collaborator_count","full_description","# Disclaimer\n\nThis is a work-in-progress project. (WIP)\n\n# Usage\n\n## 1. Build Docker Image\n[![](https://images.microbadger.com/badges/image/mohsenasm/spark-tpc-ds-performance-test-in-docker.svg)](https://hub.docker.com/r/mohsenasm/spark-tpc-ds-performance-test-in-docker)\n\nBuild docker image or pull from [docker hub](https://hub.docker.com/r/mohsenasm/spark-tpc-ds-performance-test-in-docker).\n\n```\n$ docker build . -t spark-tpc-ds\n```\n\n## 2. Create Docker Container\n\n```\n$ docker run -it -p 18080:18080 -p 4040:4040 spark-tpc-ds bash\n```\n\n## 3. Run history server\n\n```\n$ /usr/local/spark/sbin/start-history-server.sh\n```\n\nNow you can see history in browser at http://localhost:18080.\n\n## 4. Run the Script\n\nPerform the following steps to complete the execution of the script:\n\n```\n $ cd spark-tpc-ds-performance-test\n $ bin/tpcdsspark.sh\n\n==============================================\nTPC-DS On Spark Menu\n----------------------------------------------\nSETUP\n (1) Create spark tables\nRUN\n (2) Run a subset of TPC-DS queries\n (3) Run All (99) TPC-DS Queries\nCLEANUP\n (4) Cleanup\n (Q) Quit\n----------------------------------------------\nPlease enter your choice followed by [ENTER]:\n```\n\n### Setup Option: \"(1) - Create Spark Tables\"\n\nThis option creates the tables in the database name specified by `TPCDS_DBNAME` defined in `bin/tpcdsenv.sh`. The default name is `TPCDS` but can be changed if needed. The created tables are based on the pre-generated data.\n\nThe SQL statements to create the tables can be found in `src/ddl/individual`, and are created in parquet format for efficient processing. \n\n> Due to licensing restrictions, the TPCDS toolkit is not included as part of the code pattern. Instead, a pre-generated data set with 1GB scale factor is\n included in this pattern. If you want to work with a data set with larger scale factor or explore learning the full life sycle of setting up TPCDS, you can\n download the tool kit from [TPC-DS](http://www.tpc.org/tpcds) and compile in your development environment. Here are the instructions that describes how\n to compile the tool kit and generate data.\n\n\n1. Compile the toolkit\n\n ```\n unzip \n cd /tools\n make clean\n make OS=\n # (platform can be 'macos' or 'linux').\n ```\n\n2. Generate the data.\n\n ```\n cd /src/toolkit/tools\n ./dsdgen -dir -scale -verbose y -terminate n\n # data_gen_dir => The output directory where data will be generated at.\n # scale_factor => The scale factor of data.\n\n3. Generate the queries.\n\n The `dsqgen` utility in the tpcds toolkit may be used to generate the queries. Appropiate options should be passed to this utility. A typical example of its usage is:\n\n ```\n cd /tools\n ./dsqgen -VERBOSE Y -DIALECT -DIRECTORY -SCALE -OUTPUT_DIR \n ```\n\nBelow is example output for when this option is chosen.\n\n```\n==============================================\nTPC-DS On Spark Menu\n----------------------------------------------\nSETUP\n (1) Create spark tables\nRUN\n (2) Run a subset of TPC-DS queries\n (3) Run All (99) TPC-DS Queries\nCLEANUP\n (4) Cleanup\n (Q) Quit\n----------------------------------------------\nPlease enter your choice followed by [ENTER]: 1\n----------------------------------------------\n\nINFO: Creating tables. Will take a few minutes ...\nINFO: Progress : [########################################] 100%\nINFO: Spark tables created successfully..\nPress any key to continue\n```\n\n### Run Option: \"(2) - Run a subset of TPC-DS queries\"\n\n> You can also create a text file at `work/runlist.txt` with query numbers in each line,\n> then run `bin/run_the_list.sh` to run those queries.\n\nA comma separated list of queries can be specified in this option. The result of each query in the supplied list is generated in `TPCDS_WORK_DIR`, with a default directory location of `work`. The format of the result file is `query.res`.\n\nA summary file named `run_summary.txt` is also generated. It contains information about query number, execution time and number of rows returned.\n\n*Note:* The query number is a two digit number, so for query 1 the results will be in `query01.res`.\n\n*Note:* If you are debugging and running queries using this option, make sure to save `run_summary.txt` after each of your runs.\n\n```\n==============================================\nTPC-DS On Spark Menu\n----------------------------------------------\nSETUP\n (1) Create spark tables\nRUN\n (2) Run a subset of TPC-DS queries\n (3) Run All (99) TPC-DS Queries\nCLEANUP\n (4) Cleanup toolkit\n (Q) Quit\n----------------------------------------------\nPlease enter your choice followed by [ENTER]: 2\n----------------------------------------------\n\nEnter a comma separated list of queries to run (ex: 1, 2), followed by [ENTER]:\n1,2\nINFO: Checking pre-reqs for running TPC-DS queries. May take a few seconds..\nINFO: Checking pre-reqs for running TPC-DS queries is successful.\nINFO: Running TPCDS queries. Will take a few minutes depending upon the number of queries specified..\nINFO: Progress : [########################################] 100%\nINFO: TPCDS queries ran successfully. Below are the result details\nINFO: Individual result files: spark-tpc-ds-performance-test/work/query.res\nINFO: Summary file: spark-tpc-ds-performance-test/work/run_summary.txt\nPress any key to continue\n```\n\n### Run Option: \"(3) - Run all (99) TPC-DS queries\"\n\nThe only difference between this and option `(5)` is that all 99 TPC-DS queries are run instead of a subset.\n\n*Note:* If you are running this on your laptop, it can take a few hours to run all 99 TPC-DS queries.\n\n```\n==============================================\nTPC-DS On Spark Menu\n----------------------------------------------\nSETUP\n (1) Create spark tables\nRUN\n (2) Run a subset of TPC-DS queries\n (3) Run All (99) TPC-DS Queries\nCLEANUP\n (4) Cleanup toolkit\n (Q) Quit\n----------------------------------------------\nPlease enter your choice followed by [ENTER]: 3\n----------------------------------------------\nINFO: Checking pre-reqs for running TPC-DS queries. May take a few seconds..\nINFO: Checking pre-reqs for running TPC-DS queries is successful.\nINFO: Running TPCDS queries. Will take a few minutes depending upon the number of queries specified..\nINFO: Progress : [########################################] 100%\nINFO: TPCDS queries ran successfully. Below are the result details\nINFO: Individual result files: spark-tpc-ds-performance-test/work/query.res\nINFO: Summary file: spark-tpc-ds-performance-test/work/run_summary.txt\nPress any key to continue\n```\n\n### Cleanup option: \"(4) - Cleanup\"\n\nThis will clean up all of the files generated during option steps 1, 2, and 3. If you use this option, make sure to run the setup steps (1) before running queries using option 2 and 3.\n\n### Cleanup option: \"(Q) - Quit\"\n\nThis will exit the script.\n","has_starred","hub_user","immutable_tags_settings",{"_70":11,"_71":72},"enabled","rules",[73],".*","is_automated","permissions",{"_77":17,"_78":11,"_79":11},"read","write","admin","user","storage_size",9361445585,"orgData","recentTags",[86],{"_87":39,"_88":89,"_90":91,"_92":93,"_94":95,"_45":112,"_113":89,"_114":28,"_115":51,"_52":116,"_117":118,"_119":103,"_120":112,"_121":61,"_122":17},"content_type","creator",1544231,"full_size",1598648028,"id",61139866,"images",[96],{"_97":98,"_99":100,"_101":43,"_102":103,"_104":105,"_106":107,"_108":43,"_109":-5,"_110":91,"_58":61,"_111":-5},"architecture","amd64","digest","sha256:efe193e3c8d884e78f874ab75bc7ed12eed4c47ce254db6924b89f7a362f22df","features","last_pulled","2026-09-16T15:55:11.984830735Z","last_pushed","2019-07-13T20:48:56Z","os","linux","os_features","os_version","size","variant","2019-07-13T20:48:55.80586Z","last_updater","last_updater_username","media_type","latest","repository",7363458,"tag_last_pulled","tag_last_pushed","tag_status","v2","sbxKitMetadata","isPotentialSpam","isHubSbxKitsEnabled","displaySecurityMetadata","tagAttestations","tagsHaveAttestations",[],"routes/_layout.r.$namespace.$reponame.tags",{"_33":132,"_139":140,"_128":149,"_150":11},{"_35":133,"_37":134,"_40":41,"_42":43,"_44":11,"_45":46,"_47":48,"_49":135,"_52":30,"_27":28,"_53":54,"_55":39,"_56":57,"_58":59,"_60":61,"_62":-5,"_63":57,"_64":65,"_66":11,"_67":28,"_68":136,"_74":11,"_75":138,"_80":28,"_81":82},[],[39],[51],{"_70":11,"_71":137},[73],{"_77":17,"_78":11,"_79":11},"tags",{"_141":59,"_142":-5,"_143":-5,"_144":145},"count","next","previous","results",[146],{"_87":39,"_88":89,"_90":91,"_92":93,"_94":147,"_45":112,"_113":89,"_114":28,"_115":51,"_52":116,"_117":118,"_119":103,"_120":112,"_121":61,"_122":17},[148],{"_97":98,"_99":100,"_101":43,"_102":103,"_104":105,"_106":107,"_108":43,"_109":-5,"_110":91,"_58":61,"_111":-5},[],"canDeleteTags","actionData","errors"]

mohsenasm/spark-tpc-ds-performance-test-in-docker

Tags for mohsenasm/spark-tpc-ds-performance-test-in-docker