nanawel/phrawler repository overview

git clone <this repo URL> phrawler
cd phrawler
composer install

$ ./bin/console app:run https://crawler-test.com
Run started on 2026-04-07T18:36:05+02:00
REQUESTS STATUS (Incomplete/Complete) I | Queued: 0 | No rate limit
https://crawler-test.com/ | 200 | 41 KiB | 586ms | Referer: (none) | 411 link(s) | Mem: 40 MiB
REQUESTS STATUS (Incomplete/Complete) IIIIIIIIII | Queued: 393 | No rate limit
https://crawler-test.com/mobile/separate_desktop | 200 | 2 KiB | 110ms | Referer: https://crawler-test.com/ | 1 link(s) | Mem: 40 MiB
https://crawler-test.com/mobile/separate_desktop_with_different_links_out | 200 | 2 KiB | 113ms | Referer: https://crawler-test.com/ | 9 link(s) | Mem: 40 MiB
https://crawler-test.com/mobile/separate_desktop_with_different_title | 200 | 2 KiB | 114ms | Referer: https://crawler-test.com/ | 1 link(s) | Mem: 40 MiB
https://crawler-test.com/mobile/separate_desktop_with_mobile_not_subdomain | 200 | 2 KiB | 113ms | Referer: https://crawler-test.com/ | 1 link(s) | Mem: 40 MiB
https://crawler-test.com/mobile/separate_desktop_with_different_wordcount | 200 | 2 KiB | 114ms | Referer: https://crawler-test.com/ | 1 link(s) | Mem: 40 MiB
https://crawler-test.com/mobile/separate_desktop_with_different_h1 | 200 | 2 KiB | 114ms | Referer: https://crawler-test.com/ | 1 link(s) | Mem: 40 MiB
...
Run completed on 2026-04-07T18:37:01+02:00
Crawled URLs: 1049
Duration    : 00:56

$ ./bin/console app:run -h
Description:
  Crawl target URL using async requests (faster)

Usage:
  app:run:async [options] [--] <url>
  app:run
  app:run:await-any

Arguments:
  url                                                The target URL.

Options:
  -N, --max-requests=MAX-REQUESTS                    Maximum total number of requests to send [default: 0]
  -m, --url-max-crawls=URL-MAX-CRAWLS                Allow the same URL to be crawled this number of times at most. [default: 1]
  -p, --max-parallel=MAX-PARALLEL                    Maximum number of requests to send in parallel [default: 10]
  -r, --max-redirect=MAX-REDIRECT                    Max redirect per request [default: 2]
  -R, --max-retry=MAX-RETRY                          Max retry per request [default: 1]
  -E, --max-req-per-second=MAX-REQ-PER-SECOND        Max requests per second. Integer value or a "N/T" value to use a ramp-up period of T seconds to reach N req/s [default: 0]
  -a, --basic-auth=BASIC-AUTH                        Basic auth to use. Specify as user:password format [default: ""]
  -A, --crawl-assets=CRAWL-ASSETS                    Also crawl assets [default: 1]
      --crawl-assets-ext=CRAWL-ASSETS-EXT            The extensions of assets to crawl (list of values separated by commas) [default: "css,js,jpg,png,jpeg,gif,webp,woff,woff2,ttf,ico"]
      --req-header=REQ-HEADER                        Header to pass with each request as Header=Value (repeat to add more) (multiple values allowed)
      --connect-timeout=CONNECT-TIMEOUT              Connect timeout (in seconds) [default: 10]
      --transfer-timeout=TRANSFER-TIMEOUT            Transfer timeout (in seconds) [default: 10]
      --inactivity-timeout=INACTIVITY-TIMEOUT        Inactivity timeout (in seconds) [default: 10]
      --ignore-invalid-cert[=IGNORE-INVALID-CERT]    Ignore invalid SSL/TLS certificate [default: 0]
      --ignore-query[=IGNORE-QUERY]                  Ignore query when deduplicating found URLs [default: 1]
      --ignore-fragment[=IGNORE-FRAGMENT]            Ignore fragment when deduplicating found URLs (recommended) [default: 1]
  -U, --user-agent=USER-AGENT                        The user-agent to use when sending requests [default: "php/phrawler"]
  -H, --har-report-enabled=HAR-REPORT-ENABLED        Enable HAR reporter [default: 1]
      --sqlite-report-enabled=SQLITE-REPORT-ENABLED  Enable SQLite reporter [default: 1]
      --db-report-dsn=DB-REPORT-DSN                  Database reporter DSN (leave empty to disable). Must be PDO-compatible. Extended example (all values are optional): "mysql:host=localhost;port=3306;dbname=phrawler;user=phrawler;password=foo;charset=utf8mb4;useCompression=on" [default: ""]
      --report-batch-size=REPORT-BATCH-SIZE          Default batch size for report (applies to SQLite and database). Use "0" to disable batch writing and write only once at the end. [default: 50]
  -h, --help                                         Display help for the given command. When no command is given display help for the list command
      --silent                                       Do not output any message
  -q, --quiet                                        Only errors are displayed. All other output is suppressed
  -V, --version                                      Display this application version
      --ansi|--no-ansi                               Force (or disable --no-ansi) ANSI output
  -n, --no-interaction                               Do not ask any interactive question
  -e, --env=ENV                                      The Environment name. [default: "prod"]
      --no-debug                                     Switch off debug mode.
      --profile                                      Enables profiling (requires debug).
  -v|vv|vvv, --verbose                               Increase the verbosity of messages: 1 for normal output, 2 for more verbose output and 3 for debug

mkdir reports
podman run --rm -v ./reports:/app/var/reports docker.io/nanawel/phrawler app:run -p20 https://crawler-test.com

services:
  phrawler:
    image: nanawel/phrawler
    environment:
      PHP_MEMORY_LIMIT: 3G
      PHRAWLER_OPT_MAX_PARALLEL: 20
      PHRAWLER_OPT_MAX_CRAWL_PER_URL: 20
      PHRAWLER_OPT_DB_REPORT_DSN: 'mysql:host=db;dbname=phrawler;user=phrawler;password=phrawler'
    volumes:
      - './logs:/app/var/log'
      - './reports:/app/var/reports'
    depends_on:
      - db

  db:
    image: mariadb:11
    ports:
      - '3306:3306'
    environment:
      MYSQL_USER: phrawler
      MYSQL_PASSWORD: phrawler
      MYSQL_DATABASE: phrawler
      MARIADB_ALLOW_EMPTY_ROOT_PASSWORD: 1

# Create the volume folders first for the right permissions
mkdir -p logs reports

# Start the "db" container (the "phrawler" one will exit immediately)
# You might need to wait up to 1 mn for the container to be ready, otherwise you'll get a "Connection refused" when
# trying to run the crawler in the next step.
docker compose up -d

# Run your crawl, here overriding the default PHRAWLER_OPT_MAX_CRAWL_PER_URL specified in the docker-compose.yml above.
docker compose run --rm -u $(id -u):$(id -g) -e PHRAWLER_OPT_MAX_CRAWL_PER_URL=10 phrawler app:run:async https://crawler-test.com

# Then explore the results
echo 'select * from phrawler.pageurl' | docker compose exec -T db mariadb -u phrawler -pphrawler | less -Sn