# Custom configuration for StormCrawler
# This is used to override the default values from crawler-default.xml and provide additional ones
# for your custom components.
# Use this file with the parameter -config when launching your extension of ConfigurableTopology.
# This file does not contain all the key values but only the most frequently used ones. See crawler-default.xml for an extensive list.

config:
  topology.workers: 1
  topology.message.timeout.secs: 6000
  topology.max.spout.pending: 1000
  topology.debug: false

  storm.log.dir: "/tmp/storm-logs"
  storm.workers.artifacts.dir: "/tmp/workers-artifacts/"

  fetcher.threads.number: 1

  # give 2gb to the workers
  worker.heap.memory.mb: 2048

  # mandatory when using Flux
  topology.kryo.register:
    - com.digitalpebble.stormcrawler.Metadata

  # metadata to transfer to the outlinks
  # used by Fetcher for redirections, sitemapparser, etc...
  # these are also persisted for the parent document (see below)
  metadata.transfer:
  # - customMetadataName
    - event
    - trace
    - url.path
  # lists the metadata to persist to storage
  # these are not transfered to the outlinks
  # these are to keep in urls table in metadata column
  metadata.persist:
  # - _redirTo
  # - error.cause
  # - error.source
  # - isSitemap
  # - isFeed
   - warcs
  # - event
   - url.path
   - trace
   - filter
   - discoveryDate
   - lastProcessedDate
   - clickCount
   - selSessionDur
   - proxyDur
   - traceDur

  http.agent.name: "Prototeam Archiver"
  http.agent.version: "0.1"
  http.agent.description: "LANL's RL Prototeam crawler"
  http.agent.url: ""
  http.agent.email: ""
  http.skip.robots: true


  http.protocol.implementation: "gov.lanl.crawler.RemoteDriverProtocol"
  https.protocol.implementation: "gov.lanl.crawler.RemoteDriverProtocol"


  # for crawling through a warc proxy:
  #http.proxy.host: 127.0.0.1
  http.proxy.host: 127.0.0.1
  #http.proxy.host: 172.17.0.1
  http.proxy.port: 8050
  http.proxy.dir: "/home/occam/trace-archiver/warcs/warcstore"
  http.cert.dir: "/home/occam/trace-archiver/certs"
  warcprox.exec.dir: "/usr/bin/"
  #warcprox.dedup.path: "/home/occam/trace-archiver/warcprox-dedup.db"
  python.exec.dir: "/usr/bin/python"
  # storm

  #selenium.addresses: "http://172.17.0.1:4444/wd/hub"
  selenium.addresses: "http://localhost:4444/wd/hub"
  #selenium.addresses: "http://tracer-browser:4444/wd/hub"
  selenium.capabilities: {MaxInstances: 50, takesScreenshot: false, unexpectedAlertBehaviour: accept, loadImages: true, javascriptEnabled: true, browserName : chrome, chromeOptions: {args: ['user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36']}, version : 3.11.0-californium}
 # selenium.instances.num: 1
 # selenium.setScriptTimeout: 1000
  selenium.implicitlyWait: 3000



  # for crawling through a proxy with Basic authentication:
  # http.proxy.user:
  # http.proxy.pass:

  # FetcherBolt queue dump : comment out to activate
  # if a file exists on the worker machine with the corresponding port number
  # the FetcherBolt will log the content of its internal queues to the logs
  # fetcherbolt.queue.debug.filepath: "/tmp/fetcher-dump-{port}
  # time bucket to use for the metrics sent by the Fetcher
  fetcher.metrics.time.bucket.secs: 60

  #parsefilters.config.file: "parsefilters.json"
  #urlfilters.config.file: "urlfilters.json"
  navigationfilters.config.file: "boundary-filters3.json"
  #navigationfilters.config.file: "/traces/bfilters.json"
  # revisit a page daily (value in minutes)
  # set it to -1 to never refetch a page //not working for mysql
  # portal urls should not be refetched by second crawler
  fetchInterval.default: 432000

  # revisit a page with a fetch error after 2 hours (value in minutes)
  # set it to -1 to never refetch a page
  fetchInterval.fetch.error: 432000

  # never revisit a page with an error (or set a value in minutes)
  fetchInterval.error: 432000

  # custom fetch interval to be used when a document has the key/value in its metadata
  # and has been fetched succesfully (value in minutes)
  # fetchInterval.isFeed=true: 10

  # configuration for the classes extending AbstractIndexerBolt
  # indexer.md.filter: "someKey=aValue"
  indexer.url.fieldname: "url"
  indexer.text.fieldname: "content"
  indexer.canonical.name: "canonical"
  indexer.md.mapping:
    - parse.title=title
    - parse.keywords=keywords
    - parse.description=description
    - domain=domain

  # Metrics consumers:
  topology.metrics.consumer.register:
     - class: "org.apache.storm.metric.LoggingMetricsConsumer"
       parallelism.hint: 1
  mysql.nosubtrace: "false"
  mysql.url: "jdbc:mysql://localhost:3306/portals?autoReconnect=true&useSSL=false&useUnicode=true&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=UTC"
  mysql.table: "urls"
  mysql.user: "tracer"
  mysql.password: "password"
  mysql.buffer.size: 2
  mysql.min.query.interval: 5000
