[{"_1":2,"_175":-5,"_176":-5},"loaderData",{"_3":4,"_13":14,"_20":21,"_154":155},"root",{"_5":-5,"_6":-7,"_7":8,"_9":-7,"_10":11,"_12":-7},"currentUser","primaryEmail","siteVerification","6LeIxAcTAAAAAJcZVRqyHh71UMIEGNQ_MXjiZKhI","toast","enableDev",false,"incidentStatus","routes/_layout",{"_15":-5,"_16":17,"_18":17,"_19":11},"username","isAskGordonEnabled",true,"isUsingNewTemplate","showMinimalLanding","routes/_layout.r.$namespace.$reponame",{"_22":23,"_24":-5,"_25":11,"_26":11,"_27":28,"_29":30,"_31":-5,"_32":11,"_33":34,"_83":84,"_107":108,"_147":-5,"_148":11,"_149":17,"_150":11,"_151":-5,"_152":153},"canonicalUrl","https://hub.docker.com/r/dbhi/htq/tags","mcpServer","canSeeDockerfile","canSeeAutobuilds","namespace","dbhi","reponame","htq","cloudBuildSource","isAutobuildsAvailable","repo",{"_35":36,"_37":38,"_40":41,"_42":43,"_44":11,"_45":46,"_47":48,"_49":50,"_52":30,"_27":28,"_53":54,"_55":-5,"_56":57,"_58":59,"_60":61,"_62":-5,"_63":57,"_64":65,"_66":11,"_67":28,"_68":69,"_74":17,"_75":76,"_80":28,"_81":82},"categories",[],"content_types",[39],"image","date_registered","2014-12-28T14:45:23.608886Z","description","","is_private","last_updated","2015-11-15T20:32:52.225363Z","last_modified","2024-10-16T13:48:34.145251Z","media_types",[51],"application/vnd.docker.distribution.manifest.v1+prettyjws","name","pull_count",422,"repository_type","star_count",0,"status",1,"status_description","active","source","collaborator_count","full_description","# HTTP Task Queue (htq)\n\nThe motivation for this project is to provide a standard interface for sending a request in the background to a service that performs some task with a standard mechanism for signaling it to be *cancelled*.\n\nIn this context, a *task* is classified as something that may take longer than what is appropriate for a typical HTTP response or something that is allowed to be eventually completed. Examples include executing a database query, running an analysis on some data, and injesting/scraping data from websites or other services. One side effect of the client-server model is that the server may not be aware if the client aborts the request. The server will continue to perform the task using up resources that other tasks or request handlers could be using. The mechanism for signaling a cancellation is for a subsequent DELETE request to be sent to the service which can be handled to interrupt the first request. This of course requires services to support the DELETE method and implement the logic for cancelling the task being performed. See below for a working example of a [service](#service-example) implementing this interface.\n\nFor a detailed introduction, read the [tutorial](#tutorial) below.\n\n## Docker\n\n*Requires a Redis 2.4+ container.*\n\nRun the server on local port 5000.\n\n```\ndocker run -d --link redis:redis --publish 5000:5000 dbhi/htq server\n```\n\nRun a worker.\n\n```\ndocker run -d --link redis:redis dbhi/htq worker\n```\n\n## Manual Install\n\n### Dependencies\n\n- Python 3.3+\n- Redis 2.4+\n\n```\npip install htq\n```\n\n## Command-line Interface\n\n```\n$ htq -h\nHTTP Task Queue (htq) command-line interface\n\nUsage:\n htq server [--host ] [--port ] [--redis ] [--debug]\n htq worker [--threads ] [--redis ] [--debug]\n\nOptions:\n -h --help Show this screen.\n -v --version Show version.\n --debug Turns on debug logging.\n --host Host of the HTTP service [default: localhost].\n --port Port of the HTTP service [default: 5000].\n --redis Host/port of the Redis server [default: localhost:6379].\n --threads Number of threads a worker should spawn [default: 10].\n```\n\nRun the server for the HTTP REST interface.\n\n```\nhtq server\n```\n\nRun the worker to send requests and receive responses.\n\n```\nhtq worker\n```\n\n## API\n\n*Request data must be JSON-encoded and include the `Content-Type: application/json` header.*\n\n- `GET /` - Gets all queued requests.\n- `POST /` - Sends (queues) a request\n- `GET //` - Gets a request by UUID\n- `DELETE //` - Cancels a request, deleting it's response if already received\n- `GET //response/` - Gets a request's response if it has been received\n- `DELETE //response/` - Delete a request's response to clear up space\n\n### Request Attributes\n\nPOST data is a JSON-encoded object with the one or more of the following attributes:\n\n- `url` (required) - URL of the request\n- `method` - Request method. Defaults to `GET` or `POST` if `data` is supplied.\n- `data` - Request data in bytes, i.e. encoded for the defined `Content-Type`\n- `headers` - Dict of request headers\n- `timeout` - Seconds to wait before timing out the request.\n- `id` - Unique identifier for the request to support automatic cancellation of a previously queued request with the same `id`.\n\nSee examples below in the tutorial.\n\n## Tutorial\n\nStart the HTQ server:\n\n```bash\n$ htq server\nStarting htq REST server...\n * Running on http://localhost:5000/\n```\n\nSend a POST to the service with a JSON encoded structure of the request to be sent. This immediately returns a `303 See Other` response with the `Location` header to the request\n\n```bash\n$ curl -i -X POST -H 'Content-Type: application/json' http://localhost:5000 -d '{\"url\": \"http://httpbin.org/ip\"}'\nHTTP/1.0 303 SEE OTHER\nContent-Type: text/html; charset=utf-8\nContent-Length: 0\nLocation: http://localhost:5000/a936e1a1-68d8-4433-a0c0-4f4b2111670d/\nServer: Werkzeug/0.9.6 Python/3.4.1\nDate: Mon, 29 Sep 2014 17:18:49 GMT\n```\n\nSee what the request resource looks like (use the URL from the `Location` header in your output):\n\n```bash\n$ curl -i http://localhost:5000/a936e1a1-68d8-4433-a0c0-4f4b2111670d/\nHTTP/1.0 200 OK\nContent-Type: application/json\nContent-Length: 185\nLink: ; rel=\"response\",\n ; rel=\"self\",\n ; rel=\"status\"\nServer: Werkzeug/0.9.6 Python/3.4.1\nDate: Mon, 29 Sep 2014 17:26:05 GMT\n\n{\n \"timeout\": 60,\n \"status\": \"queued\",\n \"data\": null,\n \"headers\": {},\n \"url\": \"http://httpbin.org/ip\",\n \"method\": \"get\",\n \"uuid\": \"a936e1a1-68d8-4433-a0c0-4f4b2111670d\",\n \"time\": 1412011537783\n}\n```\n\nThe above response shows the details of the request such as the URL, method, headers, and request data. Additionally metadata has been captured when the request was queued including the UUID, the time (in milliseconds) when the request was queued, the status of the request and a timeout. The `Link` header shows the related links from this resource including one to the response and the status. The status link is a lightweight:\n\n\n```bash\n$ curl -i http://localhost:5000/a936e1a1-68d8-4433-a0c0-4f4b2111670d/status/\nHTTP/1.0 200 OK\nContent-Type: application/json\nContent-Length: 20\nServer: Werkzeug/0.9.6 Python/3.4.1\nDate: Mon, 29 Sep 2014 17:30:21 GMT\n\n{\"status\": \"queued\"}\n```\n\nIf we were to `curl` the response link, it would block until the response is ready. Since we have not started a worker yet, this would be forever. Let's start a worker to actual send the request. Execute this in a separate shell:\n\n```bash\n$ htq worker --debug\nStarted 10 workers...\n[a936e1a1-68d8-4433-a0c0-4f4b2111670d] sending request...\n[a936e1a1-68d8-4433-a0c0-4f4b2111670d] response received\n```\n\nStarting the worker daemon immediately starts consuming the queue and sending the requests. As you can see, the one sent above has been sent and the response received. Let's check the status of our request.\n\n```bash\n$ curl -i http://localhost:5000/a936e1a1-68d8-4433-a0c0-4f4b2111670d/status/\nHTTP/1.0 200 OK\nContent-Type: application/json\nContent-Length: 21\nServer: Werkzeug/0.9.6 Python/3.4.1\nDate: Mon, 29 Sep 2014 17:36:56 GMT\n\n{\"status\": \"success\"}\n```\n\nSuccess! Now let's use the link to the response itself.\n\n```bash\n$ curl -i http://localhost:5000/a936e1a1-68d8-4433-a0c0-4f4b2111670d/response/\nHTTP/1.0 200 OK\nContent-Type: application/json\nContent-Length: 19291\nLink: ; rel=\"request\",\n ; rel=\"self\"\nServer: Werkzeug/0.9.6 Python/3.4.1\nDate: Mon, 29 Sep 2014 17:38:22 GMT\n\n{\n \"status\": \"success\",\n \"time\": 1412012019457,\n \"code\": 200,\n \"reason\": \"OK\",\n \"elapsed\": 79.81,\n \"uuid\": \"62db48a4-e511-4c8e-9c11-32b39758d1ff\",\n \"headers\": {\n \"Age\": \"0\",\n \"Content-Length\": \"32\",\n \"Connection\": \"Keep-Alive\",\n \"Access-Control-Allow-Origin\": \"*\",\n \"Server\": \"gunicorn/18.0\",\n \"Access-Control-Allow-Credentials\": \"true\",\n \"Date\": \"Mon, 29 Sep 2014 17:50:29 GMT\",\n \"Content-Type\": \"application/json\"\n },\n \"data\": \"{\\n \\\"origin\\\": \\\"159.14.243.254\\\"\\n}\"\n}\n```\n\nThe response contains all the elements of an HTTP response including code, reason, headers, and the data (which has been removed for brevity). In addition, the time (in milliseconds) the response was received and the elapsed time (in milliseconds) join the UUID and status metadata.\n\n### Canceling a request\n\nHTQ defines an interface for services to implement for allowing requests to be canceled. For example, if I send a request that is taking longer than I expect (delayed for 30 seconds):\n\n```bash\n$ curl -i -X POST -H Content-Type:application/json http://localhost:5000 -d '{\"url\": \"http://httpbin.org/delay/30\"}'\nHTTP/1.0 303 SEE OTHER\nContent-Type: text/html; charset=utf-8\nContent-Length: 0\nLocation: http://localhost:5000/1686e1b7-3b05-4d45-95e8-caf934f540aa/\nServer: Werkzeug/0.9.6 Python/3.4.1\nDate: Mon, 29 Sep 2014 18:00:28 GMT\n```\n\nThen I can send a DELETE to the request URL:\n\n```bash\n$ curl -i -X DELETE http://localhost:5000/1686e1b7-3b05-4d45-95e8-caf934f540aa/\nHTTP/1.0 204 NO CONTENT\nContent-Type: text/html; charset=utf-8\nContent-Length: 0\nServer: Werkzeug/0.9.6 Python/3.4.1\nDate: Mon, 29 Sep 2014 18:00:41 GMT\n```\n\nNow if we check the status, we should see the status has changed to `canceled` (the response is also empty).\n\n```bash\n$ curl -i http://localhost:5000/9619b267-760d-4f0a-9f15-eb8ad99cd1c4/status/\nHTTP/1.0 200 OK\nContent-Type: application/json\nContent-Length: 22\nServer: Werkzeug/0.9.6 Python/3.4.1\nDate: Mon, 29 Sep 2014 18:01:23 GMT\n\n{\"status\": \"canceled\"}\n```\n\nInternally this interrupts the request, but also sends a DELETE request to the endpoint (in this `http://httpbin.org/delay/30`). Implementors of services can support the DELETE request to cancel the underlying processing that is occurring. Of course this is specific to the underlying task being performed, but this simple service-level contract provides a consistent mechanism for signaling the the cancellation.\n\n## Service Example\n\nBelow is a working example of a service that implements the interface HTQ requires for receiving the DELETE to signal the cancellation. This service takes a JSON-encoded statement and parameters and executes them in a local PostgreSQL instance (for simplicity, the database settings are hard-coded).\n\n```python\nimport json\nimport logging\nimport psycopg2\nfrom flask import Flask, abort, request\n\nlogger = logging.getLogger(__name__)\nlogger.setLevel(logging.DEBUG)\nhandler = logging.StreamHandler()\nlogger.addHandler(handler)\n\napp = Flask(__name__)\n\n# Database connection settings\nsettings = {\n 'dbname': 'example',\n 'host': 'localhost',\n}\n\n# Currently running tasks by UUID mapped to the\n# connection process ID\ntasks = {}\n\n\n@app.route('//', methods=['POST'])\ndef run(uuid):\n data = request.json\n\n conn = psycopg2.connect(**settings)\n c = conn.cursor()\n\n # Store the PID of the connection execute this task\n pid = conn.get_backend_pid()\n tasks[uuid] = pid\n\n try:\n logger.debug('[{}] executing query for {}...'.format(pid, uuid))\n c.execute(data['statement'], data.get('parameters'))\n except:\n conn.cancel()\n abort(500)\n finally:\n tasks.pop(uuid, None)\n\n return json.dumps(c.fetchall()), 200\n\n\n@app.route('//', methods=['DELETE'])\ndef cancel(uuid):\n if uuid not in tasks:\n abort(404)\n\n pid = tasks.pop(uuid)\n\n # Open new connection to cancel the query\n conn = psycopg2.connect(**settings)\n c = conn.cursor()\n\n c.execute('select pg_terminate_backend(%s)', (pid,))\n logger.debug('[{}] canceled query for {}'.format(pid, uuid))\n\n return '', 204\n\n\nif __name__ == '__main__':\n app.run(threaded=True)\n```\n\nThe interaction looks as follows:\n\n```bash\n% python example_service.py\n * Running on http://127.0.0.1:5000/\n[57819] executing query for abc123...\n[57819] canceled query for abc123\n127.0.0.1 - - [30/Sep/2014 11:16:01] \"DELETE /abc123/ HTTP/1.1\" 204 -\n127.0.0.1 - - [30/Sep/2014 11:16:02] \"POST /abc123/ HTTP/1.1\" 500 -\n```\n\nA POST request was first sent to execute a query. A DELETE request was sent shortly after to cancel the query and returns. When the POST request does respond it is a 500 complaining the connection was closed (which is what we wanted).\n","has_starred","hub_user","immutable_tags_settings",{"_70":11,"_71":72},"enabled","rules",[73],".*","is_automated","permissions",{"_77":17,"_78":11,"_79":11},"read","write","admin","user","storage_size",392597416,"orgData",{"_85":86,"_87":88,"_89":90,"_91":28,"_92":93,"_94":95,"_96":97,"_98":99,"_100":101,"_102":103,"_104":43,"_105":43,"_106":17},"id","179b7bd4bd5e48d293f152a1f403c00d","uuid","179b7bd4-bd5e-48d2-93f1-52a1f403c00d","type","organization","orgname","full_name","Department of Biomedical and Health Informatics","location","Philadelphia, PA","company","The Children's Hospital of Philadelphia","profile_url","http://dbhi.chop.edu/","date_joined","2014-12-28T14:33:29.651561Z","gravatar_url","https://www.gravatar.com/avatar/641d4c98abc8fca6731ad9ada22c6d61?s=80&r=g&d=mm","gravatar_email","badge","is_active","recentTags",[109],{"_110":39,"_111":112,"_113":114,"_115":116,"_85":117,"_118":119,"_45":134,"_135":136,"_137":138,"_139":51,"_52":140,"_141":142,"_143":125,"_144":134,"_145":61,"_146":17},"content_type","creator",178198,"digest","sha256:88cb6ca90b28ca6b7a117489977fd48e45aed38f8a07aebbfe425735d7b2ba18","full_size",392560110,92386,"images",[120],{"_121":122,"_113":114,"_123":43,"_124":125,"_126":127,"_128":43,"_129":43,"_130":-5,"_131":132,"_58":61,"_133":-5},"architecture","amd64","features","last_pulled","2026-09-16T16:42:35.478735217Z","last_pushed","2015-11-15T20:32:51Z","os","os_features","os_version","size",392560366,"variant","2015-11-15T20:32:56.015904Z","last_updater",364857,"last_updater_username","jeffmax","media_type","latest","repository",107429,"tag_last_pulled","tag_last_pushed","tag_status","v2","sbxKitMetadata","isPotentialSpam","isHubSbxKitsEnabled","displaySecurityMetadata","tagAttestations","tagsHaveAttestations",[],"routes/_layout.r.$namespace.$reponame.tags",{"_33":156,"_163":164,"_152":173,"_174":11},{"_35":157,"_37":158,"_40":41,"_42":43,"_44":11,"_45":46,"_47":48,"_49":159,"_52":30,"_27":28,"_53":54,"_55":-5,"_56":57,"_58":59,"_60":61,"_62":-5,"_63":57,"_64":65,"_66":11,"_67":28,"_68":160,"_74":17,"_75":162,"_80":28,"_81":82},[],[39],[51],{"_70":11,"_71":161},[73],{"_77":17,"_78":11,"_79":11},"tags",{"_165":59,"_166":-5,"_167":-5,"_168":169},"count","next","previous","results",[170],{"_110":39,"_111":112,"_113":114,"_115":116,"_85":117,"_118":171,"_45":134,"_135":136,"_137":138,"_139":51,"_52":140,"_141":142,"_143":125,"_144":134,"_145":61,"_146":17},[172],{"_121":122,"_113":114,"_123":43,"_124":125,"_126":127,"_128":43,"_129":43,"_130":-5,"_131":132,"_58":61,"_133":-5},[],"canDeleteTags","actionData","errors"]

dbhi/htq

Tags for dbhi/htq