|
C
|
ca
|
2024-04-27 04:45:25
|
2024-04-27 04:45:26
|
|
|
C
|
ca_ab
|
2024-04-27 04:23:42
|
2024-04-27 04:23:43
|
|
D>
04:13:16 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:4806016/legislature yet
|
ca_ab_calgary
|
|
2024-04-27 04:13:16
|
AssertionError: No councillors found
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_ab_calgary/people.py", line 15, in scrape
assert len(councillors), "No councillors found"
AssertionError: No councillors found
|
D>
04:17:43 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/province:ab/executive yet
|
ca_ab_candidates
|
|
2024-04-27 04:17:43
|
pupa.exceptions.UnresolvedIdError: cannot resolve pseudo id to Post: ~{"label": "Innisfail-Sylvan Lake", "organization__clas…
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 307, in do_handle
report['import'] = self.do_import(juris, args)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 211, in do_import
report.update(membership_importer.import_directory(datadir))
File "/app/.heroku/src/pupa/pupa/importers/base.py", line 190, in import_directory
return self.import_data(json_stream())
File "/app/.heroku/src/pupa/pupa/importers/base.py", line 227, in import_data
obj_id, what = self.import_item(data)
File "/app/.heroku/src/pupa/pupa/importers/base.py", line 247, in import_item
data = self.prepare_for_db(data)
File "/app/.heroku/src/pupa/pupa/importers/memberships.py", line 50, in prepare_for_db
data['post_id'] = self.post_importer.resolve_json_id(data['post_id'])
File "/app/.heroku/src/pupa/pupa/importers/base.py", line 165, in resolve_json_id
raise UnresolvedIdError(errmsg)
pupa.exceptions.UnresolvedIdError: cannot resolve pseudo id to Post: ~{"label": "Innisfail-Sylvan Lake", "organization__classification": "executive", "role": "MLA"}
|
D>
04:24:02 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:4811061/legislature yet
|
ca_ab_edmonton
|
|
2024-04-27 04:24:02
|
AssertionError: No councillors found
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_ab_edmonton/people.py", line 14, in scrape
assert len(councillors), "No councillors found"
AssertionError: No councillors found
|
D>
04:13:09 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:4819012/legislature yet
|
ca_ab_grande_prairie
|
|
2024-04-27 04:13:09
|
KeyError: 'name'
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/utils.py", line 394, in scrape
if not self.is_valid_row(row):
File "/app/scrapers/utils.py", line 348, in is_valid_row
return row["name"] not in empty
KeyError: 'name'
|
|
C
|
ca_ab_grande_prairie_county_no_1
|
2024-04-27 04:22:03
|
2024-04-27 04:22:03
|
|
D>
04:17:52 WARNING scrapelib: sleeping for 10 seconds before retry
04:18:03 WARNING scrapelib: sleeping for 20 seconds before retry
04:18:23 WARNING scrapelib: sleeping for 40 seconds before retry
|
ca_ab_lethbridge
|
2022-12-08 04:08:13
|
2024-04-27 04:19:03
|
scrapelib.HTTPError: 410 while retrieving https://docs.google.com/spreadsheets/d/1OnHJq_j-r3R4MMkRQ5ahNkApuDp1NpTYn4UVTTNGY5…
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/utils.py", line 379, in scrape
reader = self.csv_reader(
File "/app/scrapers/utils.py", line 227, in csv_reader
response = self.get(url, **kwargs)
File "/app/scrapers/utils.py", line 184, in get
return super().get(*args, verify=SSL_VERIFY, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 602, in request
raise HTTPError(resp)
scrapelib.HTTPError: 410 while retrieving https://docs.google.com/spreadsheets/d/1OnHJq_j-r3R4MMkRQ5ahNkApuDp1NpTYn4UVTTNGY5c/pub?gid=908195318&single=true&output=csv
|
|
D>
|
ca_ab_strathcona_county
|
2023-12-01 04:03:03
|
2024-04-27 21:12:36
|
KeyError: 'name'
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/utils.py", line 394, in scrape
if not self.is_valid_row(row):
File "/app/scrapers/utils.py", line 348, in is_valid_row
return row["name"] not in empty
KeyError: 'name'
|
D>
04:19:09 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:4816037/legislature yet
|
ca_ab_wood_buffalo
|
|
2024-04-27 04:19:09
|
IndexError: list index out of range
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_ab_wood_buffalo/people.py", line 15, in scrape
mayor_url = page.xpath('//li[@id="pageid1075"]/div/a/@href')[0]
IndexError: list index out of range
|
|
C
|
ca_bc
|
2024-04-27 04:09:00
|
2024-04-27 04:09:00
|
|
D>
04:11:40 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:5909052/legislature yet
|
ca_bc_abbotsford
|
|
2024-04-27 04:11:40
|
AssertionError: No councillors found
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_bc_abbotsford/people.py", line 19, in scrape
assert len(councillors), "No councillors found"
AssertionError: No councillors found
|
D>
04:11:18 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:5915025/legislature yet
|
ca_bc_burnaby
|
|
2024-04-27 04:11:18
|
AssertionError: No councillors found
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_bc_burnaby/people.py", line 13, in scrape
assert len(councillors), "No councillors found"
AssertionError: No councillors found
|
D>
04:25:57 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:5915034/legislature yet
|
ca_bc_coquitlam
|
|
2024-04-27 04:25:57
|
scrapelib.HTTPError: 404 while retrieving https://www.coquitlam.ca/city-hall/mayor-and-council/mayor-and-council.aspx
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_bc_coquitlam/people.py", line 11, in scrape
page = self.lxmlize(COUNCIL_PAGE)
File "/app/scrapers/utils.py", line 192, in lxmlize
response = self.get(url, cookies=cookies)
File "/app/scrapers/utils.py", line 184, in get
return super().get(*args, verify=SSL_VERIFY, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 602, in request
raise HTTPError(resp)
scrapelib.HTTPError: 404 while retrieving https://www.coquitlam.ca/city-hall/mayor-and-council/mayor-and-council.aspx
|
|
C
|
ca_bc_kelowna
|
2024-04-27 04:07:08
|
2024-04-27 04:07:08
|
|
|
D>
|
ca_bc_langley
|
2022-10-18 04:01:08
|
2024-04-27 04:24:20
|
scrapelib.HTTPError: 404 while retrieving https://opendata.arcgis.com/datasets/daa3ea0f01d24e9b80d837cf2178eb71_0.csv
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/utils.py", line 379, in scrape
reader = self.csv_reader(
File "/app/scrapers/utils.py", line 227, in csv_reader
response = self.get(url, **kwargs)
File "/app/scrapers/utils.py", line 184, in get
return super().get(*args, verify=SSL_VERIFY, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 602, in request
raise HTTPError(resp)
scrapelib.HTTPError: 404 while retrieving https://opendata.arcgis.com/datasets/daa3ea0f01d24e9b80d837cf2178eb71_0.csv
|
D>
04:11:53 WARNING scrapelib: sleeping for 10 seconds before retry
04:12:03 WARNING scrapelib: sleeping for 20 seconds before retry
04:12:23 WARNING scrapelib: sleeping for 40 seconds before retry
04:13:03 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/province:bc/government yet
|
ca_bc_municipalities
|
|
2024-04-27 04:13:03
|
scrapelib.HTTPError: 403 while retrieving https://www.civicinfo.bc.ca/people
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_bc_municipalities/people.py", line 67, in scrape
list_page = self.lxmlize(LIST_PAGE)
File "/app/scrapers/utils.py", line 192, in lxmlize
response = self.get(url, cookies=cookies)
File "/app/scrapers/utils.py", line 184, in get
return super().get(*args, verify=SSL_VERIFY, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 602, in request
raise HTTPError(resp)
scrapelib.HTTPError: 403 while retrieving https://www.civicinfo.bc.ca/people
|
|
C
|
ca_bc_municipalities_candidates
|
2024-04-27 04:22:00
|
2024-04-27 04:22:01
|
|
D>
04:00:34 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:5915029/legislature yet
|
ca_bc_new_westminster
|
|
2024-04-27 04:00:34
|
scrapelib.HTTPError: 404 while retrieving https://opendata.newwestcity.ca/downloads/councillor-contact-information/councillo…
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/utils.py", line 379, in scrape
reader = self.csv_reader(
File "/app/scrapers/utils.py", line 227, in csv_reader
response = self.get(url, **kwargs)
File "/app/scrapers/utils.py", line 184, in get
return super().get(*args, verify=SSL_VERIFY, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 602, in request
raise HTTPError(resp)
scrapelib.HTTPError: 404 while retrieving https://opendata.newwestcity.ca/downloads/councillor-contact-information/councillor_contacts.csv
|
|
D>
|
ca_bc_richmond
|
2023-04-29 04:03:23
|
2024-04-27 04:16:26
|
scrapelib.HTTPError: 404 while retrieving https://www.richmond.ca/contact/departments/council.htm
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_bc_richmond/people.py", line 12, in scrape
contact_page = self.lxmlize(CONTACT_URL)
File "/app/scrapers/utils.py", line 192, in lxmlize
response = self.get(url, cookies=cookies)
File "/app/scrapers/utils.py", line 184, in get
return super().get(*args, verify=SSL_VERIFY, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 602, in request
raise HTTPError(resp)
scrapelib.HTTPError: 404 while retrieving https://www.richmond.ca/contact/departments/council.htm
|
|
C
|
ca_bc_saanich
|
2024-04-27 04:13:12
|
2024-04-27 04:13:12
|
|
D>
04:10:11 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:5915004/legislature yet
|
ca_bc_surrey
|
|
2024-04-27 04:10:11
|
AssertionError: No members found
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_bc_surrey/people.py", line 14, in scrape
assert len(members), "No members found"
AssertionError: No members found
|
|
C
|
ca_bc_vancouver
|
2024-04-27 04:17:50
|
2024-04-27 04:17:50
|
|
D>
04:11:41 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:5917034/legislature yet
|
ca_bc_victoria
|
|
2024-04-27 04:11:41
|
KeyError: 'primary role'
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/utils.py", line 413, in scrape
role = row["primary role"]
KeyError: 'primary role'
|
|
C
|
ca_candidates
|
2024-04-27 04:24:18
|
2024-04-27 04:24:19
|
|
|
D>
|
ca_mb
|
2024-04-27 04:10:10
|
2024-04-27 10:17:50
|
SystemExit: 1
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_mb/people.py", line 39, in scrape
page = self.lxmlize(url)
File "/app/scrapers/utils.py", line 192, in lxmlize
response = self.get(url, cookies=cookies)
File "/app/scrapers/utils.py", line 184, in get
return super().get(*args, verify=SSL_VERIFY, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 579, in request
resp = super().request(
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 404, in request
resp = super().request(
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 232, in request
return super().request(
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 122, in request
resp = super().request(
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 589, in request
resp = self.send(prep, **send_kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 703, in send
r = adapter.send(request, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/requests/adapters.py", line 486, in send
resp = conn.urlopen(
File "/app/.heroku/python/lib/python3.8/site-packages/urllib3/connectionpool.py", line 715, in urlopen
httplib_response = self._make_request(
File "/app/.heroku/python/lib/python3.8/site-packages/urllib3/connectionpool.py", line 416, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/app/.heroku/python/lib/python3.8/site-packages/urllib3/connection.py", line 244, in request
super(HTTPConnection, self).request(method, url, body=body, headers=headers)
File "/app/.heroku/python/lib/python3.8/http/client.py", line 1256, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/app/.heroku/python/lib/python3.8/http/client.py", line 1302, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/app/.heroku/python/lib/python3.8/http/client.py", line 1251, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/app/.heroku/python/lib/python3.8/http/client.py", line 1011, in _send_output
self.send(msg)
File "/app/.heroku/python/lib/python3.8/http/client.py", line 951, in send
self.connect()
File "/app/.heroku/python/lib/python3.8/site-packages/urllib3/connection.py", line 205, in connect
conn = self._new_conn()
File "/app/.heroku/python/lib/python3.8/site-packages/urllib3/connection.py", line 174, in _new_conn
conn = connection.create_connection(
File "/app/.heroku/python/lib/python3.8/site-packages/urllib3/util/connection.py", line 85, in create_connection
sock.connect(sa)
File "/app/.heroku/python/lib/python3.8/site-packages/gunicorn/workers/base.py", line 196, in handle_abort
sys.exit(1)
SystemExit: 1
|
D>
04:07:05 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/province:mb/executive yet
|
ca_mb_candidates
|
|
2024-04-27 04:07:05
|
pupa.exceptions.UnresolvedIdError: cannot resolve pseudo id to Post: ~{"label": "River East", "organization__classification"…
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 307, in do_handle
report['import'] = self.do_import(juris, args)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 211, in do_import
report.update(membership_importer.import_directory(datadir))
File "/app/.heroku/src/pupa/pupa/importers/base.py", line 190, in import_directory
return self.import_data(json_stream())
File "/app/.heroku/src/pupa/pupa/importers/base.py", line 227, in import_data
obj_id, what = self.import_item(data)
File "/app/.heroku/src/pupa/pupa/importers/base.py", line 247, in import_item
data = self.prepare_for_db(data)
File "/app/.heroku/src/pupa/pupa/importers/memberships.py", line 50, in prepare_for_db
data['post_id'] = self.post_importer.resolve_json_id(data['post_id'])
File "/app/.heroku/src/pupa/pupa/importers/base.py", line 165, in resolve_json_id
raise UnresolvedIdError(errmsg)
pupa.exceptions.UnresolvedIdError: cannot resolve pseudo id to Post: ~{"label": "River East", "organization__classification": "executive", "role": "candidate"}
|
|
D>
|
ca_mb_winnipeg
|
2022-10-25 04:10:28
|
2024-04-27 04:07:24
|
AssertionError: No councillors found
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_mb_winnipeg/people.py", line 22, in scrape
assert len(councillors), "No councillors found"
AssertionError: No councillors found
|
D>
04:19:15 WARNING pupa: validation of Membership 4f21a39a-044d-11ef-9183-42c196ffa11a failed: 1 validation errors:
Value 'town hall.weedon@hsfqc.ca' for field '' does not match regular expression '\A([A-Za-z0-9._\'-]+)@(?:[A-Za-z0-9-]+\.)+[A-Za-z]{2,}\Z'
|
ca_municipalities
|
|
2024-04-27 04:19:15
|
Value 'town hall.weedon@hsfqc.ca' for field '' does not match regular expression '\A([A-Za-z0-9._\'-]+)@(?:[A-Za-z0-9-]+\.)+…
Traceback (most recent call last):
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 175, in validate
validator.validate(self.as_dict(), schema)
File "/app/.heroku/python/lib/python3.8/site-packages/validictory/validator.py", line 620, in validate
raise MultipleValidationError(self._errors)
validictory.validator.MultipleValidationError: 1 validation errors:
Value 'town hall.weedon@hsfqc.ca' for field '' does not match regular expression '\A([A-Za-z0-9._\'-]+)@(?:[A-Za-z0-9-]+\.)+[A-Za-z]{2,}\Z'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 104, in do_scrape
self.save_object(obj)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 93, in save_object
self.save_object(obj)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 89, in save_object
raise ve
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 85, in save_object
obj.validate()
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 177, in validate
raise ScrapeValueError('validation of {} {} failed: {}'.format(
pupa.exceptions.ScrapeValueError: validation of Membership 4f21a39a-044d-11ef-9183-42c196ffa11a failed: 1 validation errors:
Value 'town hall.weedon@hsfqc.ca' for field '' does not match regular expression '\A([A-Za-z0-9._\'-]+)@(?:[A-Za-z0-9-]+\.)+[A-Za-z]{2,}\Z'
|
|
C
|
ca_nb
|
2024-04-27 04:11:16
|
2024-04-27 04:11:16
|
|
|
D>
|
ca_nb_fredericton
|
2023-06-18 22:29:30
|
2024-04-27 04:19:17
|
scrapelib.HTTPError: 404 while retrieving https://www.fredericton.ca/en/city-hall/city-council-committees/mayor-council
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_nb_fredericton/people.py", line 9, in scrape
page = self.lxmlize(COUNCIL_PAGE)
File "/app/scrapers/utils.py", line 192, in lxmlize
response = self.get(url, cookies=cookies)
File "/app/scrapers/utils.py", line 184, in get
return super().get(*args, verify=SSL_VERIFY, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 602, in request
raise HTTPError(resp)
scrapelib.HTTPError: 404 while retrieving https://www.fredericton.ca/en/city-hall/city-council-committees/mayor-council
|
D>
04:11:34 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:1307022/legislature yet
|
ca_nb_moncton
|
|
2024-04-27 04:11:34
|
scrapelib.HTTPError: 404 while retrieving https://opendata.arcgis.com/datasets/d81d30cf2b0d4bf7ae7aea5b0acc9d5f_0.csv
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/utils.py", line 379, in scrape
reader = self.csv_reader(
File "/app/scrapers/utils.py", line 227, in csv_reader
response = self.get(url, **kwargs)
File "/app/scrapers/utils.py", line 184, in get
return super().get(*args, verify=SSL_VERIFY, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 602, in request
raise HTTPError(resp)
scrapelib.HTTPError: 404 while retrieving https://opendata.arcgis.com/datasets/d81d30cf2b0d4bf7ae7aea5b0acc9d5f_0.csv
|
D>
04:24:08 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/province:nb/government yet
|
ca_nb_municipalities
|
|
2024-04-27 04:24:08
|
AssertionError: No list items found
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_nb_municipalities/people.py", line 65, in scrape
assert len(list_links), "No list items found"
AssertionError: No list items found
|
|
C
|
ca_nb_saint_john
|
2024-04-27 04:23:45
|
2024-04-27 04:23:45
|
|
|
C
|
ca_nl
|
2024-04-27 04:11:37
|
2024-04-27 04:11:37
|
|
|
D>
|
ca_nl_st_john_s
|
2022-10-26 04:07:32
|
2024-04-27 04:11:33
|
AssertionError: No councillors found
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_nl_st_john_s/people.py", line 15, in scrape
assert len(councillors), "No councillors found"
AssertionError: No councillors found
|
|
C
|
ca_ns
|
2024-04-27 04:28:30
|
2024-04-27 04:28:30
|
|
D>
04:16:24 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:1217030/legislature yet
|
ca_ns_cape_breton
|
|
2024-04-27 04:16:24
|
pupa.exceptions.SameNameError: multiple people with same name "Lorne Green" in Jurisdiction - must provide birth_date to dis…
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 307, in do_handle
report['import'] = self.do_import(juris, args)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 207, in do_import
report.update(person_importer.import_directory(datadir))
File "/app/.heroku/src/pupa/pupa/importers/base.py", line 190, in import_directory
return self.import_data(json_stream())
File "/app/.heroku/src/pupa/pupa/importers/base.py", line 226, in import_data
for json_id, data in self._prepare_imports(data_items):
File "/app/.heroku/src/pupa/pupa/importers/people.py", line 33, in _prepare_imports
raise SameNameError(name)
pupa.exceptions.SameNameError: multiple people with same name "Lorne Green" in Jurisdiction - must provide birth_date to disambiguate
|
D>
04:16:31 WARNING scrapelib: sleeping for 10 seconds before retry
04:16:41 WARNING scrapelib: sleeping for 20 seconds before retry
04:17:01 WARNING scrapelib: sleeping for 40 seconds before retry
04:17:41 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:1209034/legislature yet
|
ca_ns_halifax
|
|
2024-04-27 04:17:41
|
scrapelib.HTTPError: 403 while retrieving https://www.halifax.ca/city-hall/districts-councillors
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_ns_halifax/people.py", line 14, in scrape
page = self.lxmlize(COUNCIL_PAGE)
File "/app/scrapers/utils.py", line 192, in lxmlize
response = self.get(url, cookies=cookies)
File "/app/scrapers/utils.py", line 184, in get
return super().get(*args, verify=SSL_VERIFY, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 602, in request
raise HTTPError(resp)
scrapelib.HTTPError: 403 while retrieving https://www.halifax.ca/city-hall/districts-councillors
|
|
C
|
ca_on
|
2024-04-27 04:32:07
|
2024-04-27 04:32:08
|
|
D>
04:23:46 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:3518005/legislature yet
|
ca_on_ajax
|
|
2024-04-27 04:23:46
|
IndexError: list index out of range
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_on_ajax/people.py", line 27, in scrape
voice = councillor.xpath('.//p[contains(.,"Cel")]/text()')[1]
IndexError: list index out of range
|
D>
04:16:27 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:3512005/legislature yet
|
ca_on_belleville
|
|
2024-04-27 04:16:27
|
scrapelib.HTTPError: 404 while retrieving https://www.belleville.ca/city-hall/page/city-council
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_on_belleville/people.py", line 14, in scrape
page = self.lxmlize(COUNCIL_PAGE)
File "/app/scrapers/utils.py", line 192, in lxmlize
response = self.get(url, cookies=cookies)
File "/app/scrapers/utils.py", line 184, in get
return super().get(*args, verify=SSL_VERIFY, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 602, in request
raise HTTPError(resp)
scrapelib.HTTPError: 404 while retrieving https://www.belleville.ca/city-hall/page/city-council
|
|
C
|
ca_on_brampton
|
2024-04-27 04:17:48
|
2024-04-27 04:17:48
|
|
|
C
|
ca_on_brantford
|
2024-04-27 04:16:29
|
2024-04-27 04:16:29
|
|
D>
04:26:05 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:3524002/legislature yet
|
ca_on_burlington
|
|
2024-04-27 04:26:05
|
scrapelib.HTTPError: 404 while retrieving https://opendata.arcgis.com/datasets/f2c404c667904f03be38443cbd474cd3_0.csv
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/utils.py", line 379, in scrape
reader = self.csv_reader(
File "/app/scrapers/utils.py", line 227, in csv_reader
response = self.get(url, **kwargs)
File "/app/scrapers/utils.py", line 184, in get
return super().get(*args, verify=SSL_VERIFY, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 602, in request
raise HTTPError(resp)
scrapelib.HTTPError: 404 while retrieving https://opendata.arcgis.com/datasets/f2c404c667904f03be38443cbd474cd3_0.csv
|
D>
04:10:13 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:3521024/legislature yet
|
ca_on_caledon
|
|
2024-04-27 04:10:13
|
IndexError: list index out of range
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_on_caledon/people.py", line 11, in scrape
node = page.xpath('//td[@rowspan="2"]')[0]
IndexError: list index out of range
|
|
C
|
ca_on_cambridge
|
2024-04-27 04:16:06
|
2024-04-27 04:16:06
|
|
D>
04:28:40 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:3536020/legislature yet
|
ca_on_chatham_kent
|
|
2024-04-27 04:28:40
|
scrapelib.HTTPError: 404 while retrieving https://www.chatham-kent.ca/local-government/council/council-members
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_on_chatham_kent/people.py", line 16, in scrape
page = self.lxmlize(COUNCIL_PAGE)
File "/app/scrapers/utils.py", line 192, in lxmlize
response = self.get(url, cookies=cookies)
File "/app/scrapers/utils.py", line 184, in get
return super().get(*args, verify=SSL_VERIFY, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 602, in request
raise HTTPError(resp)
scrapelib.HTTPError: 404 while retrieving https://www.chatham-kent.ca/local-government/council/council-members
|
|
C
|
ca_on_fort_erie
|
2024-04-27 04:22:10
|
2024-04-27 04:22:10
|
|
|
C
|
ca_on_georgina
|
2024-04-27 04:07:16
|
2024-04-27 04:07:16
|
|
|
C
|
ca_on_greater_sudbury
|
2024-04-27 04:11:49
|
2024-04-27 04:11:49
|
|
|
D>
|
ca_on_grimsby
|
2024-01-12 04:22:13
|
2024-04-27 04:24:03
|
scrapelib.HTTPError: 404 while retrieving https://niagaraopendata.ca/dataset/fe096749-6ca8-4ae7-b80e-dc682a698759/resource/8…
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/utils.py", line 379, in scrape
reader = self.csv_reader(
File "/app/scrapers/utils.py", line 227, in csv_reader
response = self.get(url, **kwargs)
File "/app/scrapers/utils.py", line 184, in get
return super().get(*args, verify=SSL_VERIFY, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 602, in request
raise HTTPError(resp)
scrapelib.HTTPError: 404 while retrieving https://niagaraopendata.ca/dataset/fe096749-6ca8-4ae7-b80e-dc682a698759/resource/892a4873-cbe7-4c32-9b70-9f364467955e/download/grimsbycouncil20160210.csv
|
|
C
|
ca_on_guelph
|
2024-04-27 04:27:33
|
2024-04-27 04:27:33
|
|
D>
04:10:16 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:3528018/legislature yet
|
ca_on_haldimand_county
|
|
2024-04-27 04:10:16
|
AssertionError: No councillors found
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_on_haldimand_county/people.py", line 14, in scrape
assert len(councillors), "No councillors found"
AssertionError: No councillors found
|
D>
04:17:46 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:3525005/legislature yet
|
ca_on_hamilton
|
|
2024-04-27 04:17:46
|
scrapelib.HTTPError: 404 while retrieving https://www.hamilton.ca/council-committee/mayor-council/city-councillors
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_on_hamilton/people.py", line 9, in scrape
page = self.lxmlize(COUNCIL_PAGE)
File "/app/scrapers/utils.py", line 192, in lxmlize
response = self.get(url, cookies=cookies)
File "/app/scrapers/utils.py", line 184, in get
return super().get(*args, verify=SSL_VERIFY, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 602, in request
raise HTTPError(resp)
scrapelib.HTTPError: 404 while retrieving https://www.hamilton.ca/council-committee/mayor-council/city-councillors
|
D>
04:10:24 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/cd:3540/legislature yet
|
ca_on_huron
|
|
2024-04-27 04:10:24
|
pupa.exceptions.UnresolvedIdError: cannot resolve pseudo id to Post: ~{"label": "Huron (seat 7)", "organization__classificat…
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 307, in do_handle
report['import'] = self.do_import(juris, args)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 211, in do_import
report.update(membership_importer.import_directory(datadir))
File "/app/.heroku/src/pupa/pupa/importers/base.py", line 190, in import_directory
return self.import_data(json_stream())
File "/app/.heroku/src/pupa/pupa/importers/base.py", line 227, in import_data
obj_id, what = self.import_item(data)
File "/app/.heroku/src/pupa/pupa/importers/base.py", line 247, in import_item
data = self.prepare_for_db(data)
File "/app/.heroku/src/pupa/pupa/importers/memberships.py", line 50, in prepare_for_db
data['post_id'] = self.post_importer.resolve_json_id(data['post_id'])
File "/app/.heroku/src/pupa/pupa/importers/base.py", line 165, in resolve_json_id
raise UnresolvedIdError(errmsg)
pupa.exceptions.UnresolvedIdError: cannot resolve pseudo id to Post: ~{"label": "Huron (seat 7)", "organization__classification": "legislature", "role": "Councillor"}
|
|
C
|
ca_on_king
|
2024-04-27 04:16:10
|
2024-04-27 04:16:10
|
|
|
C
|
ca_on_kingston
|
2024-04-27 04:10:25
|
2024-04-27 04:10:25
|
|
|
C
|
ca_on_kitchener
|
2024-04-27 04:07:25
|
2024-04-27 04:07:26
|
|
D>
04:28:36 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/cd:3538/legislature yet
|
ca_on_lambton
|
|
2024-04-27 04:28:36
|
AssertionError: No councillors found
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_on_lambton/people.py", line 15, in scrape
assert len(councillors), "No councillors found"
AssertionError: No councillors found
|
|
C
|
ca_on_lincoln
|
2024-04-27 04:00:41
|
2024-04-27 04:00:41
|
|
D>
04:19:06 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:3539036/legislature yet
|
ca_on_london
|
|
2024-04-27 04:19:06
|
scrapelib.HTTPError: 404 while retrieving https://apps.london.ca/OpenData/CSV/Council.csv
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/utils.py", line 379, in scrape
reader = self.csv_reader(
File "/app/scrapers/utils.py", line 227, in csv_reader
response = self.get(url, **kwargs)
File "/app/scrapers/utils.py", line 184, in get
return super().get(*args, verify=SSL_VERIFY, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 602, in request
raise HTTPError(resp)
scrapelib.HTTPError: 404 while retrieving https://apps.london.ca/OpenData/CSV/Council.csv
|
D>
04:11:31 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:3519036/legislature yet
|
ca_on_markham
|
|
2024-04-27 04:11:31
|
IndexError: list index out of range
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_on_markham/people.py", line 13, in scrape
mayor_url = page.xpath('//a[contains(text(), "Office of the Mayor")]/@href')[0]
IndexError: list index out of range
|
D>
04:11:52 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:3524009/legislature yet
|
ca_on_milton
|
|
2024-04-27 04:11:52
|
AssertionError: No councillors found
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_on_milton/people.py", line 14, in scrape
assert len(councillors), "No councillors found"
AssertionError: No councillors found
|
|
D>
|
ca_on_mississauga
|
2024-03-19 04:07:47
|
2024-04-27 04:09:06
|
IndexError: list index out of range
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_on_mississauga/people.py", line 17, in scrape
yield self.councillor_data(councillor_url.attrib["href"])
File "/app/scrapers/ca_on_mississauga/people.py", line 35, in councillor_data
photo = page.xpath(
IndexError: list index out of range
|
|
C
|
ca_on_newmarket
|
2024-04-27 04:07:27
|
2024-04-27 04:07:27
|
|
|
C
|
ca_on_niagara
|
2024-04-27 04:19:08
|
2024-04-27 04:19:08
|
|
|
C
|
ca_on_niagara_on_the_lake
|
2024-04-27 04:11:44
|
2024-04-27 04:11:44
|
|
|
C
|
ca_on_north_dumfries
|
2024-04-27 04:00:36
|
2024-04-27 04:00:36
|
|
D>
04:13:13 WARNING pupa: validation of CanadianPerson 77519434-044c-11ef-9183-42c196ffa11a failed: 1 validation errors:
Value 'Sean Oâ\x80\x99Meara' for field '<obj>.name' does not match regular expression 'regex.Regex('\\A(?!(?:Chair|Commissioner|Conseiller|Councillor|Deputy|Dr|Hon|M|Maire|Mayor|Miss|Mme|Mr|Mrs|Ms|Regional|Warden)\\b)(?:(?:(?:\\p{Lu}\\.)+|\\p{Lu}+|(?:Jr|Rev|Sr|St)\\.|da|de|den|der|la|van|von|[("](?:\\p{Lu}+|\\p{Lu}\\p{Ll}*(?:-\\p{Lu}\\p{Ll}*)*)[)"]|(?:D\'|d\'|De|de|Des|Di|Du|L\'|La|Le|Mac|Mc|O\'|San|Van|Vander?|vanden)?\\p{Lu}\\p{Ll}+|\\p{Lu}\\p{Ll}+Anne?|Marie\\p{Lu}\\p{Ll}+|Ch\'ng|Prud\'homme|D!ONNE|IsaBelle|Ya\'ara)(?:\'|-| - | ))+(?:(?:\\p{Lu}\\.)+|\\p{Lu}+|(?:Jr|Rev|Sr|St)\\.|da|de|den|der|la|van|von|[("](?:\\p{Lu}+|\\p{Lu}\\p{Ll}*(?:-\\p{Lu}\\p{Ll}*)*)[)"]|(?:D\'|d\'|De|de|Des|Di|Du|L\'|La|Le|Mac|Mc|O\'|San|Van|Vander?|vanden)?\\p{Lu}\\p{Ll}+|\\p{Lu}\\p{Ll}+Anne?|Marie\\p{Lu}\\p{Ll}+|Ch\'ng|Prud\'homme|D!ONNE|IsaBelle|Ya\'ara)\\Z', flags=regex.V0)'
04:13:13 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:3524001/legislature yet
|
ca_on_oakville
|
|
2024-04-27 04:13:13
|
Value 'Sean Oâ\x80\x99Meara' for field '<obj>.name' does not match regular expression 'regex.Regex('\\A(?!(?:Chair|Commissio…
Traceback (most recent call last):
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 175, in validate
validator.validate(self.as_dict(), schema)
File "/app/.heroku/python/lib/python3.8/site-packages/validictory/validator.py", line 620, in validate
raise MultipleValidationError(self._errors)
validictory.validator.MultipleValidationError: 1 validation errors:
Value 'Sean Oâ\x80\x99Meara' for field '<obj>.name' does not match regular expression 'regex.Regex('\\A(?!(?:Chair|Commissioner|Conseiller|Councillor|Deputy|Dr|Hon|M|Maire|Mayor|Miss|Mme|Mr|Mrs|Ms|Regional|Warden)\\b)(?:(?:(?:\\p{Lu}\\.)+|\\p{Lu}+|(?:Jr|Rev|Sr|St)\\.|da|de|den|der|la|van|von|[("](?:\\p{Lu}+|\\p{Lu}\\p{Ll}*(?:-\\p{Lu}\\p{Ll}*)*)[)"]|(?:D\'|d\'|De|de|Des|Di|Du|L\'|La|Le|Mac|Mc|O\'|San|Van|Vander?|vanden)?\\p{Lu}\\p{Ll}+|\\p{Lu}\\p{Ll}+Anne?|Marie\\p{Lu}\\p{Ll}+|Ch\'ng|Prud\'homme|D!ONNE|IsaBelle|Ya\'ara)(?:\'|-| - | ))+(?:(?:\\p{Lu}\\.)+|\\p{Lu}+|(?:Jr|Rev|Sr|St)\\.|da|de|den|der|la|van|von|[("](?:\\p{Lu}+|\\p{Lu}\\p{Ll}*(?:-\\p{Lu}\\p{Ll}*)*)[)"]|(?:D\'|d\'|De|de|Des|Di|Du|L\'|La|Le|Mac|Mc|O\'|San|Van|Vander?|vanden)?\\p{Lu}\\p{Ll}+|\\p{Lu}\\p{Ll}+Anne?|Marie\\p{Lu}\\p{Ll}+|Ch\'ng|Prud\'homme|D!ONNE|IsaBelle|Ya\'ara)\\Z', flags=regex.V0)'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 104, in do_scrape
self.save_object(obj)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 89, in save_object
raise ve
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 85, in save_object
obj.validate()
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 177, in validate
raise ScrapeValueError('validation of {} {} failed: {}'.format(
pupa.exceptions.ScrapeValueError: validation of CanadianPerson 77519434-044c-11ef-9183-42c196ffa11a failed: 1 validation errors:
Value 'Sean Oâ\x80\x99Meara' for field '<obj>.name' does not match regular expression 'regex.Regex('\\A(?!(?:Chair|Commissioner|Conseiller|Councillor|Deputy|Dr|Hon|M|Maire|Mayor|Miss|Mme|Mr|Mrs|Ms|Regional|Warden)\\b)(?:(?:(?:\\p{Lu}\\.)+|\\p{Lu}+|(?:Jr|Rev|Sr|St)\\.|da|de|den|der|la|van|von|[("](?:\\p{Lu}+|\\p{Lu}\\p{Ll}*(?:-\\p{Lu}\\p{Ll}*)*)[)"]|(?:D\'|d\'|De|de|Des|Di|Du|L\'|La|Le|Mac|Mc|O\'|San|Van|Vander?|vanden)?\\p{Lu}\\p{Ll}+|\\p{Lu}\\p{Ll}+Anne?|Marie\\p{Lu}\\p{Ll}+|Ch\'ng|Prud\'homme|D!ONNE|IsaBelle|Ya\'ara)(?:\'|-| - | ))+(?:(?:\\p{Lu}\\.)+|\\p{Lu}+|(?:Jr|Rev|Sr|St)\\.|da|de|den|der|la|van|von|[("](?:\\p{Lu}+|\\p{Lu}\\p{Ll}*(?:-\\p{Lu}\\p{Ll}*)*)[)"]|(?:D\'|d\'|De|de|Des|Di|Du|L\'|La|Le|Mac|Mc|O\'|San|Van|Vander?|vanden)?\\p{Lu}\\p{Ll}+|\\p{Lu}\\p{Ll}+Anne?|Marie\\p{Lu}\\p{Ll}+|Ch\'ng|Prud\'homme|D!ONNE|IsaBelle|Ya\'ara)\\Z', flags=regex.V0)'
|
D>
04:28:35 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:3518013/legislature yet
|
ca_on_oshawa
|
|
2024-04-27 04:28:35
|
scrapelib.HTTPError: 404 while retrieving https://www.oshawa.ca/city-hall/city-council-members.asp
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_on_oshawa/people.py", line 9, in scrape
page = self.lxmlize(COUNCIL_PAGE)
File "/app/scrapers/utils.py", line 192, in lxmlize
response = self.get(url, cookies=cookies)
File "/app/scrapers/utils.py", line 184, in get
return super().get(*args, verify=SSL_VERIFY, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 602, in request
raise HTTPError(resp)
scrapelib.HTTPError: 404 while retrieving https://www.oshawa.ca/city-hall/city-council-members.asp
|
D>
04:07:06 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:3506008/legislature yet
|
ca_on_ottawa
|
|
2024-04-27 04:07:06
|
KeyError: 'name'
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/utils.py", line 394, in scrape
if not self.is_valid_row(row):
File "/app/scrapers/utils.py", line 348, in is_valid_row
return row["name"] not in empty
KeyError: 'name'
|
D>
04:17:44 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/cd:3521/legislature yet
|
ca_on_peel
|
|
2024-04-27 04:17:44
|
scrapelib.HTTPError: 404 while retrieving https://data.peelregion.ca/media/43505/wards1822_csv.csv
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/utils.py", line 379, in scrape
reader = self.csv_reader(
File "/app/scrapers/utils.py", line 227, in csv_reader
response = self.get(url, **kwargs)
File "/app/scrapers/utils.py", line 184, in get
return super().get(*args, verify=SSL_VERIFY, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 602, in request
raise HTTPError(resp)
scrapelib.HTTPError: 404 while retrieving https://data.peelregion.ca/media/43505/wards1822_csv.csv
|
D>
04:32:09 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:3518001/legislature yet
|
ca_on_pickering
|
|
2024-04-27 04:32:09
|
AssertionError: No councillors found
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_on_pickering/people.py", line 17, in scrape
assert len(councillors), "No councillors found"
AssertionError: No councillors found
|
|
C
|
ca_on_richmond_hill
|
2024-04-27 04:11:30
|
2024-04-27 04:11:30
|
|
D>
04:29:57 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/province:on/school yet
|
ca_on_school_boards_english_public
|
|
2024-04-27 04:29:57
|
KeyError: 'district name'
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/utils.py", line 400, in scrape
elif row[key] in corrections:
KeyError: 'district name'
|
|
C
|
ca_on_st_catharines
|
2024-04-27 04:27:23
|
2024-04-27 04:27:23
|
|
D>
04:07:28 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:3558004/legislature yet
|
ca_on_thunder_bay
|
|
2024-04-27 04:07:28
|
requests.exceptions.SSLError: HTTPSConnectionPool(host='www.thunderbay.ca', port=443): Max retries exceeded with url: /en/ci…
Traceback (most recent call last):
File "/app/.heroku/python/lib/python3.8/site-packages/urllib3/connectionpool.py", line 715, in urlopen
httplib_response = self._make_request(
File "/app/.heroku/python/lib/python3.8/site-packages/urllib3/connectionpool.py", line 404, in _make_request
self._validate_conn(conn)
File "/app/.heroku/python/lib/python3.8/site-packages/urllib3/connectionpool.py", line 1058, in _validate_conn
conn.connect()
File "/app/.heroku/python/lib/python3.8/site-packages/urllib3/connection.py", line 419, in connect
self.sock = ssl_wrap_socket(
File "/app/.heroku/python/lib/python3.8/site-packages/urllib3/util/ssl_.py", line 449, in ssl_wrap_socket
ssl_sock = _ssl_wrap_socket_impl(
File "/app/.heroku/python/lib/python3.8/site-packages/urllib3/util/ssl_.py", line 493, in _ssl_wrap_socket_impl
return ssl_context.wrap_socket(sock, server_hostname=server_hostname)
File "/app/.heroku/python/lib/python3.8/ssl.py", line 500, in wrap_socket
return self.sslsocket_class._create(
File "/app/.heroku/python/lib/python3.8/ssl.py", line 1073, in _create
self.do_handshake()
File "/app/.heroku/python/lib/python3.8/ssl.py", line 1342, in do_handshake
self._sslobj.do_handshake()
ssl.SSLError: [SSL: DH_KEY_TOO_SMALL] dh key too small (_ssl.c:1131)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/app/.heroku/python/lib/python3.8/site-packages/requests/adapters.py", line 486, in send
resp = conn.urlopen(
File "/app/.heroku/python/lib/python3.8/site-packages/urllib3/connectionpool.py", line 799, in urlopen
retries = retries.increment(
File "/app/.heroku/python/lib/python3.8/site-packages/urllib3/util/retry.py", line 592, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='www.thunderbay.ca', port=443): Max retries exceeded with url: /en/city-hall/mayor-and-council.aspx (Caused by SSLError(SSLError(1, '[SSL: DH_KEY_TOO_SMALL] dh key too small (_ssl.c:1131)')))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_on_thunder_bay/people.py", line 10, in scrape
page = self.lxmlize(COUNCIL_PAGE)
File "/app/scrapers/utils.py", line 192, in lxmlize
response = self.get(url, cookies=cookies)
File "/app/scrapers/utils.py", line 184, in get
return super().get(*args, verify=SSL_VERIFY, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 579, in request
resp = super().request(
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 404, in request
resp = super().request(
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 232, in request
return super().request(
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 122, in request
resp = super().request(
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 589, in request
resp = self.send(prep, **send_kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 703, in send
r = adapter.send(request, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/requests/adapters.py", line 517, in send
raise SSLError(e, request=request)
requests.exceptions.SSLError: HTTPSConnectionPool(host='www.thunderbay.ca', port=443): Max retries exceeded with url: /en/city-hall/mayor-and-council.aspx (Caused by SSLError(SSLError(1, '[SSL: DH_KEY_TOO_SMALL] dh key too small (_ssl.c:1131)')))
|
|
C
|
ca_on_toronto
|
2024-04-27 04:00:39
|
2024-04-27 04:00:39
|
|
|
C
|
ca_on_uxbridge
|
2024-04-27 04:27:26
|
2024-04-27 04:27:26
|
|
D>
04:29:54 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:3519028/legislature yet
|
ca_on_vaughan
|
|
2024-04-27 04:29:54
|
AssertionError: No councillors found
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_on_vaughan/people.py", line 16, in scrape
assert len(councillors), "No councillors found"
AssertionError: No councillors found
|
|
C
|
ca_on_waterloo
|
2024-04-27 04:29:52
|
2024-04-27 04:29:52
|
|
D>
04:24:07 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/cd:3530/legislature yet
|
ca_on_waterloo_region
|
|
2024-04-27 04:24:07
|
KeyError: 'district name'
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/utils.py", line 399, in scrape
row[key] = corrections(row[key])
KeyError: 'district name'
|
D>
04:17:45 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:3526032/legislature yet
|
ca_on_welland
|
|
2024-04-27 04:17:45
|
scrapelib.HTTPError: 404 while retrieving https://niagaraopendata.ca/dataset/b38e2f85-bcd4-43fd-95d5-f513919514d9/resource/5…
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/utils.py", line 379, in scrape
reader = self.csv_reader(
File "/app/scrapers/utils.py", line 227, in csv_reader
response = self.get(url, **kwargs)
File "/app/scrapers/utils.py", line 184, in get
return super().get(*args, verify=SSL_VERIFY, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 602, in request
raise HTTPError(resp)
scrapelib.HTTPError: 404 while retrieving https://niagaraopendata.ca/dataset/b38e2f85-bcd4-43fd-95d5-f513919514d9/resource/590c9110-01f3-4256-8ac1-9c5cbeca88c5/download/city-of-welland-mayor-and-council-members.csv
|
|
D>
|
ca_on_wellesley
|
2024-03-19 04:38:26
|
2024-04-27 14:59:15
|
AssertionError
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_on_wellesley/people.py", line 21, in scrape
assert len(members) == 5
AssertionError
|
D>
04:17:51 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:3518009/legislature yet
|
ca_on_whitby
|
|
2024-04-27 04:17:51
|
scrapelib.HTTPError: 404 while retrieving https://www.whitby.ca/Common/PageNotFound.aspx?aspxerrorpath=/en/town-hall/council…
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_on_whitby/people.py", line 10, in scrape
page = self.lxmlize(COUNCIL_PAGE)
File "/app/scrapers/utils.py", line 192, in lxmlize
response = self.get(url, cookies=cookies)
File "/app/scrapers/utils.py", line 184, in get
return super().get(*args, verify=SSL_VERIFY, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 602, in request
raise HTTPError(resp)
scrapelib.HTTPError: 404 while retrieving https://www.whitby.ca/Common/PageNotFound.aspx?aspxerrorpath=/en/town-hall/council-members.aspx
|
|
C
|
ca_on_whitchurch_stouffville
|
2024-04-27 04:07:14
|
2024-04-27 04:07:14
|
|
|
D>
|
ca_on_windsor
|
2023-06-05 04:00:27
|
2024-04-27 20:44:26
|
scrapelib.HTTPError: 404 while retrieving https://www.citywindsor.ca/opendata/Lists/OpenData/Attachments/33/City%20Windsor%2…
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/utils.py", line 379, in scrape
reader = self.csv_reader(
File "/app/scrapers/utils.py", line 227, in csv_reader
response = self.get(url, **kwargs)
File "/app/scrapers/utils.py", line 184, in get
return super().get(*args, verify=SSL_VERIFY, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 602, in request
raise HTTPError(resp)
scrapelib.HTTPError: 404 while retrieving https://www.citywindsor.ca/opendata/Lists/OpenData/Attachments/33/City%20Windsor%20Elected%20Officials.csv
|
|
C
|
ca_pe
|
2024-04-27 04:16:05
|
2024-04-27 04:16:05
|
|
D>
04:28:41 WARNING scrapelib: sleeping for 10 seconds before retry
04:28:51 WARNING scrapelib: sleeping for 20 seconds before retry
04:29:11 WARNING scrapelib: sleeping for 40 seconds before retry
04:29:51 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:1102075/legislature yet
|
ca_pe_charlottetown
|
|
2024-04-27 04:29:51
|
scrapelib.HTTPError: 403 while retrieving https://www.charlottetown.ca/mayor___council/city_council/meet_my_councillor
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_pe_charlottetown/people.py", line 11, in scrape
page = self.lxmlize(COUNCIL_PAGE)
File "/app/scrapers/utils.py", line 192, in lxmlize
response = self.get(url, cookies=cookies)
File "/app/scrapers/utils.py", line 184, in get
return super().get(*args, verify=SSL_VERIFY, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 602, in request
raise HTTPError(resp)
scrapelib.HTTPError: 403 while retrieving https://www.charlottetown.ca/mayor___council/city_council/meet_my_councillor
|
|
C
|
ca_qc
|
2024-04-27 04:07:02
|
2024-04-27 04:07:02
|
|
|
D>
|
ca_qc_beaconsfield
|
2024-03-27 04:04:06
|
2024-04-27 04:28:38
|
AssertionError: No councillors found
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_qc_beaconsfield/people.py", line 12, in scrape
assert len(councillors), "No councillors found"
AssertionError: No councillors found
|
D>
04:22:09 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:2458007/legislature yet
|
ca_qc_brossard
|
|
2024-04-27 04:22:09
|
AssertionError: No councillors found
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_qc_brossard/people.py", line 18, in scrape
assert len(councillors), "No councillors found"
AssertionError: No councillors found
|
D>
04:11:46 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:2466058/legislature yet
|
ca_qc_cote_saint_luc
|
|
2024-04-27 04:11:46
|
AssertionError: No councillors found
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_qc_cote_saint_luc/people.py", line 13, in scrape
assert len(councillors), "No councillors found"
AssertionError: No councillors found
|
D>
04:26:09 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:2466142/legislature yet
|
ca_qc_dollard_des_ormeaux
|
|
2024-04-27 04:26:09
|
scrapelib.HTTPError: 404 while retrieving https://ville.ddo.qc.ca/en/my-municipality/members-council
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_qc_dollard_des_ormeaux/people.py", line 9, in scrape
page = self.lxmlize(COUNCIL_PAGE)
File "/app/scrapers/utils.py", line 192, in lxmlize
response = self.get(url, cookies=cookies)
File "/app/scrapers/utils.py", line 184, in get
return super().get(*args, verify=SSL_VERIFY, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 602, in request
raise HTTPError(resp)
scrapelib.HTTPError: 404 while retrieving https://ville.ddo.qc.ca/en/my-municipality/members-council
|
|
C
|
ca_qc_dorval
|
2024-04-27 04:26:04
|
2024-04-27 04:26:04
|
|
D>
04:26:10 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:2481017/legislature yet
|
ca_qc_gatineau
|
|
2024-04-27 04:26:10
|
IndexError: list index out of range
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_qc_gatineau/people.py", line 20, in scrape
p = Person(primary_org="legislature", name=names[0], district="Gatineau", role="Maire")
IndexError: list index out of range
|
D>
04:20:23 WARNING scrapelib: sleeping for 10 seconds before retry
04:20:33 WARNING scrapelib: sleeping for 20 seconds before retry
04:20:53 WARNING scrapelib: sleeping for 40 seconds before retry
04:21:33 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:2466102/legislature yet
|
ca_qc_kirkland
|
|
2024-04-27 04:21:33
|
scrapelib.HTTPError: 403 while retrieving https://www.ville.kirkland.qc.ca/portrait-municipal/conseil-municipal/elus-municip…
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_qc_kirkland/people.py", line 11, in scrape
page = self.lxmlize(COUNCIL_PAGE, "iso-8859-1")
File "/app/scrapers/utils.py", line 192, in lxmlize
response = self.get(url, cookies=cookies)
File "/app/scrapers/utils.py", line 184, in get
return super().get(*args, verify=SSL_VERIFY, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 602, in request
raise HTTPError(resp)
scrapelib.HTTPError: 403 while retrieving https://www.ville.kirkland.qc.ca/portrait-municipal/conseil-municipal/elus-municipaux
|
|
C
|
ca_qc_laval
|
2024-04-27 04:10:15
|
2024-04-27 04:10:15
|
|
D>
04:15:35 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:2425213/legislature yet
|
ca_qc_levis
|
|
2024-04-27 04:15:35
|
IndexError: list index out of range
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_qc_levis/people.py", line 15, in scrape
position, name = person.xpath("./h2/text()")[0].replace("–", "-").split(" - ")
IndexError: list index out of range
|
D>
04:26:01 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:2458227/legislature yet
|
ca_qc_longueuil
|
|
2024-04-27 04:26:01
|
scrapelib.HTTPError: 404 while retrieving https://www.longueuil.quebec/fr/conseil-ville
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_qc_longueuil/people.py", line 10, in scrape
page = self.lxmlize(COUNCIL_PAGE, "utf-8")
File "/app/scrapers/utils.py", line 192, in lxmlize
response = self.get(url, cookies=cookies)
File "/app/scrapers/utils.py", line 184, in get
return super().get(*args, verify=SSL_VERIFY, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 602, in request
raise HTTPError(resp)
scrapelib.HTTPError: 404 while retrieving https://www.longueuil.quebec/fr/conseil-ville
|
D>
04:13:34 WARNING scrapelib: got Exceeded 30 redirects. sleeping for 10 seconds before retry
04:14:00 WARNING scrapelib: got Exceeded 30 redirects. sleeping for 20 seconds before retry
04:14:36 WARNING scrapelib: got Exceeded 30 redirects. sleeping for 40 seconds before retry
04:15:33 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:2467045/legislature yet
|
ca_qc_mercier
|
|
2024-04-27 04:15:33
|
requests.exceptions.TooManyRedirects: Exceeded 30 redirects.
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_qc_mercier/people.py", line 12, in scrape
page = self.lxmlize(COUNCIL_PAGE, user_agent=CUSTOM_USER_AGENT, encoding="windows-1252")
File "/app/scrapers/utils.py", line 192, in lxmlize
response = self.get(url, cookies=cookies)
File "/app/scrapers/utils.py", line 184, in get
return super().get(*args, verify=SSL_VERIFY, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 579, in request
resp = super().request(
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 404, in request
resp = super().request(
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 232, in request
return super().request(
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 175, in request
raise exception_raised
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 122, in request
resp = super().request(
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 589, in request
resp = self.send(prep, **send_kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 725, in send
history = [resp for resp in gen]
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 725, in <listcomp>
history = [resp for resp in gen]
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 191, in resolve_redirects
raise TooManyRedirects(
requests.exceptions.TooManyRedirects: Exceeded 30 redirects.
|
D>
04:07:18 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:2466023/legislature yet
|
ca_qc_montreal
|
|
2024-04-27 04:07:18
|
scrapelib.HTTPError: 404 while retrieving https://donnees.montreal.ca/dataset/381d74ca-dadd-459f-95c9-db255b5f4480/resource/…
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/utils.py", line 379, in scrape
reader = self.csv_reader(
File "/app/scrapers/utils.py", line 227, in csv_reader
response = self.get(url, **kwargs)
File "/app/scrapers/utils.py", line 184, in get
return super().get(*args, verify=SSL_VERIFY, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 602, in request
raise HTTPError(resp)
scrapelib.HTTPError: 404 while retrieving https://donnees.montreal.ca/dataset/381d74ca-dadd-459f-95c9-db255b5f4480/resource/ce1315a3-50ee-48d0-a0f0-9bcc15f65643/download/listeelusmontreal.csv
|
D>
04:30:00 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:2466007/legislature yet
|
ca_qc_montreal_est
|
|
2024-04-27 04:30:00
|
scrapelib.HTTPError: 404 while retrieving https://ville.montreal-est.qc.ca/histoire-et-patrimoine/histoire-de-montreal-est/c…
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_qc_montreal_est/people.py", line 10, in scrape
page = self.lxmlize(COUNCIL_PAGE)
File "/app/scrapers/utils.py", line 192, in lxmlize
response = self.get(url, cookies=cookies)
File "/app/scrapers/utils.py", line 184, in get
return super().get(*args, verify=SSL_VERIFY, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 602, in request
raise HTTPError(resp)
scrapelib.HTTPError: 404 while retrieving https://ville.montreal-est.qc.ca/histoire-et-patrimoine/histoire-de-montreal-est/conseil-municipal/conseils-municipaux/
|
D>
21:09:39 WARNING pupa: validation of CanadianPerson 759a9e36-04da-11ef-9da4-ee1ea2110b14 failed: 1 validation errors:
Value 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAGQAQAAAACoxAthAAAAAnRSTlMAAHaTzTgAAAAqSURBVHja7cExAQAAAMKg9U/tbwagAAAAAAAAAAAAAAAAAAAAAAAAAIA3T7AAAZRBGFwAAAAASUVORK5CYII=' for field '<obj>.image' does not match regular expression '\A(?:(?:ftp|https?)://|\Z)'
|
ca_qc_pointe_claire
|
2024-03-12 04:56:49
|
2024-04-27 21:09:39
|
Value 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAGQAQAAAACoxAthAAAAAnRSTlMAAHaTzTgAAAAqSURBVHja7cExAQAAAMKg9U/tbwa…
Traceback (most recent call last):
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 175, in validate
validator.validate(self.as_dict(), schema)
File "/app/.heroku/python/lib/python3.8/site-packages/validictory/validator.py", line 620, in validate
raise MultipleValidationError(self._errors)
validictory.validator.MultipleValidationError: 1 validation errors:
Value 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAGQAQAAAACoxAthAAAAAnRSTlMAAHaTzTgAAAAqSURBVHja7cExAQAAAMKg9U/tbwagAAAAAAAAAAAAAAAAAAAAAAAAAIA3T7AAAZRBGFwAAAAASUVORK5CYII=' for field '<obj>.image' does not match regular expression '\A(?:(?:ftp|https?)://|\Z)'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 104, in do_scrape
self.save_object(obj)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 89, in save_object
raise ve
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 85, in save_object
obj.validate()
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 177, in validate
raise ScrapeValueError('validation of {} {} failed: {}'.format(
pupa.exceptions.ScrapeValueError: validation of CanadianPerson 759a9e36-04da-11ef-9da4-ee1ea2110b14 failed: 1 validation errors:
Value 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAZAAAAGQAQAAAACoxAthAAAAAnRSTlMAAHaTzTgAAAAqSURBVHja7cExAQAAAMKg9U/tbwagAAAAAAAAAAAAAAAAAAAAAAAAAIA3T7AAAZRBGFwAAAAASUVORK5CYII=' for field '<obj>.image' does not match regular expression '\A(?:(?:ftp|https?)://|\Z)'
|
D>
04:13:08 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:2423027/legislature yet
|
ca_qc_quebec
|
|
2024-04-27 04:13:08
|
pupa.exceptions.UnresolvedIdError: cannot resolve pseudo id to Post: ~{"label": "Loretteville\u2014Les Ch\u00e2tels", "organ…
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 307, in do_handle
report['import'] = self.do_import(juris, args)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 211, in do_import
report.update(membership_importer.import_directory(datadir))
File "/app/.heroku/src/pupa/pupa/importers/base.py", line 190, in import_directory
return self.import_data(json_stream())
File "/app/.heroku/src/pupa/pupa/importers/base.py", line 227, in import_data
obj_id, what = self.import_item(data)
File "/app/.heroku/src/pupa/pupa/importers/base.py", line 247, in import_item
data = self.prepare_for_db(data)
File "/app/.heroku/src/pupa/pupa/importers/memberships.py", line 50, in prepare_for_db
data['post_id'] = self.post_importer.resolve_json_id(data['post_id'])
File "/app/.heroku/src/pupa/pupa/importers/base.py", line 165, in resolve_json_id
raise UnresolvedIdError(errmsg)
pupa.exceptions.UnresolvedIdError: cannot resolve pseudo id to Post: ~{"label": "Loretteville\u2014Les Ch\u00e2tels", "organization__classification": "legislature", "role": "Conseiller"}
|
D>
04:27:32 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:2494068/legislature yet
|
ca_qc_saguenay
|
|
2024-04-27 04:27:32
|
IndexError: list index out of range
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_qc_saguenay/people.py", line 14, in scrape
name = mayor_page.xpath('//span/text()[contains(., "maire")]')[0].split(", ", 1)[0]
IndexError: list index out of range
|
D>
04:26:11 WARNING scrapelib: sleeping for 10 seconds before retry
04:26:21 WARNING scrapelib: sleeping for 20 seconds before retry
04:26:41 WARNING scrapelib: sleeping for 40 seconds before retry
|
ca_qc_sainte_anne_de_bellevue
|
2023-10-24 04:04:32
|
2024-04-27 04:27:22
|
scrapelib.HTTPError: 403 while retrieving https://ville.sainte-anne-de-bellevue.qc.ca/fr/199/elus-municipaux
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_qc_sainte_anne_de_bellevue/people.py", line 11, in scrape
page = self.lxmlize(COUNCIL_PAGE)
File "/app/scrapers/utils.py", line 192, in lxmlize
response = self.get(url, cookies=cookies)
File "/app/scrapers/utils.py", line 184, in get
return super().get(*args, verify=SSL_VERIFY, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 602, in request
raise HTTPError(resp)
scrapelib.HTTPError: 403 while retrieving https://ville.sainte-anne-de-bellevue.qc.ca/fr/199/elus-municipaux
|
|
C
|
ca_qc_saint_jean_sur_richelieu
|
2024-04-27 04:25:56
|
2024-04-27 04:25:56
|
|
D>
04:28:34 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:2475017/legislature yet
|
ca_qc_saint_jerome
|
|
2024-04-27 04:28:34
|
Exception: No email node in b'<div class=" inner_member">\n <div class="portrait_single">\n <img width="819" hei…
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_qc_saint_jerome/people.py", line 31, in scrape
p.add_contact("email", self.get_email(councillor))
File "/app/scrapers/utils.py", line 141, in get_email
raise Exception("No email node in {}".format(etree.tostring(node)))
Exception: No email node in b'<div class=" inner_member">\n <div class="portrait_single">\n <img width="819" height="1024" src="data:image/svg+xml,%3Csvg%20xmlns=\'http://www.w3.org/2000/svg\'%20viewBox=\'0%200%20819%201024\'%3E%3C/svg%3E" class="single_pic wp-post-image" alt="Marc Bourcier" decoding="async" fetchpriority="high" data-lazy-srcset="https://www.vsj.ca/wp-content/uploads/2019/11/0_Marc_Bourcier-819x1024.jpg 819w, https://www.vsj.ca/wp-content/uploads/2019/11/0_Marc_Bourcier-240x300.jpg 240w, https://www.vsj.ca/wp-content/uploads/2019/11/0_Marc_Bourcier-768x960.jpg 768w, https://www.vsj.ca/wp-content/uploads/2019/11/0_Marc_Bourcier-1229x1536.jpg 1229w, https://www.vsj.ca/wp-content/uploads/2019/11/0_Marc_Bourcier-500x625.jpg 500w, https://www.vsj.ca/wp-content/uploads/2019/11/0_Marc_Bourcier-800x1000.jpg 800w, https://www.vsj.ca/wp-content/uploads/2019/11/0_Marc_Bourcier-1280x1600.jpg 1280w, https://www.vsj.ca/wp-content/uploads/2019/11/0_Marc_Bourcier.jpg 1536w" data-lazy-sizes="(max-width: 819px) 100vw, 819px" data-lazy-src="https://www.vsj.ca/wp-content/uploads/2019/11/0_Marc_Bourcier-819x1024.jpg"/><noscript><img width="819" height="1024" src="https://www.vsj.ca/wp-content/uploads/2019/11/0_Marc_Bourcier-819x1024.jpg" class="single_pic wp-post-image" alt="Marc Bourcier" decoding="async" fetchpriority="high" srcset="https://www.vsj.ca/wp-content/uploads/2019/11/0_Marc_Bourcier-819x1024.jpg 819w, https://www.vsj.ca/wp-content/uploads/2019/11/0_Marc_Bourcier-240x300.jpg 240w, https://www.vsj.ca/wp-content/uploads/2019/11/0_Marc_Bourcier-768x960.jpg 768w, https://www.vsj.ca/wp-content/uploads/2019/11/0_Marc_Bourcier-1229x1536.jpg 1229w, https://www.vsj.ca/wp-content/uploads/2019/11/0_Marc_Bourcier-500x625.jpg 500w, https://www.vsj.ca/wp-content/uploads/2019/11/0_Marc_Bourcier-800x1000.jpg 800w, https://www.vsj.ca/wp-content/uploads/2019/11/0_Marc_Bourcier-1280x1600.jpg 1280w, https://www.vsj.ca/wp-content/uploads/2019/11/0_Marc_Bourcier.jpg 1536w" sizes="(max-width: 819px) 100vw, 819px"/></noscript> </div>\n <div class="info_member">\n <div class="district">\n Maire </div>\n <h2 class="entry-title">Marc Bourcier</h2>\n <div class="phone">\n 450 436-1512, poste 3021 </div>\n \n <div class="wp-block-list mail">\n <a class="space-a" href="https://www.vsj.ca/cdn-cgi/l/email-protection#81ece0e8f3e8e4c1f7f2ebafe2e0"><span class="__cf_email__" data-cfemail="fb969a9289929ebb8d8891d5989a">[email protected]</span></a>\n </div>\n <div class="wp-block-list wysiwyg">\n<ul><li>Président du comité exécutif</li><li>Membre d’office de tous les comités ainsi que toutes les commissions du conseil</li><li>Administrateur de la Municipalité régionale de comté de la Rivière-du-Nord (MRC)</li><li>Vice-Président de la Régie intermunicipale de l’Aréna régional de la Rivière-du-Nord</li></ul>\n</div> </div>\n </div> \n '
|
|
D>
|
ca_qc_senneville
|
2022-09-06 04:41:24
|
2024-04-27 04:24:24
|
scrapelib.HTTPError: 404 while retrieving https://www.senneville.ca/fr/7/conseil-municipal
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_qc_senneville/people.py", line 9, in scrape
page = self.lxmlize(COUNCIL_PAGE)
File "/app/scrapers/utils.py", line 192, in lxmlize
response = self.get(url, cookies=cookies)
File "/app/scrapers/utils.py", line 184, in get
return super().get(*args, verify=SSL_VERIFY, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 602, in request
raise HTTPError(resp)
scrapelib.HTTPError: 404 while retrieving https://www.senneville.ca/fr/7/conseil-municipal
|
D>
04:00:37 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:2443027/legislature yet
|
ca_qc_sherbrooke
|
|
2024-04-27 04:00:37
|
AssertionError: No councillors found
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_qc_sherbrooke/people.py", line 12, in scrape
assert len(councillors), "No councillors found"
AssertionError: No councillors found
|
D>
04:24:29 WARNING scrapelib: sleeping for 10 seconds before retry
04:24:39 WARNING scrapelib: sleeping for 20 seconds before retry
04:24:59 WARNING scrapelib: sleeping for 40 seconds before retry
04:25:40 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:2464008/legislature yet
|
ca_qc_terrebonne
|
|
2024-04-27 04:25:40
|
scrapelib.HTTPError: 500 while retrieving https://terrebonne.ca/fr/10/Conseil_municipal
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_qc_terrebonne/people.py", line 9, in scrape
page = self.lxmlize(COUNCIL_PAGE, "utf-8")
File "/app/scrapers/utils.py", line 192, in lxmlize
response = self.get(url, cookies=cookies)
File "/app/scrapers/utils.py", line 184, in get
return super().get(*args, verify=SSL_VERIFY, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/requests/sessions.py", line 602, in get
return self.request("GET", url, **kwargs)
File "/app/.heroku/python/lib/python3.8/site-packages/scrapelib/__init__.py", line 602, in request
raise HTTPError(resp)
scrapelib.HTTPError: 500 while retrieving https://terrebonne.ca/fr/10/Conseil_municipal
|
D>
04:09:10 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:2437067/legislature yet
|
ca_qc_trois_rivieres
|
|
2024-04-27 04:09:10
|
AssertionError: No councillors found
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_qc_trois_rivieres/people.py", line 16, in scrape
assert len(members), "No councillors found"
AssertionError: No councillors found
|
|
D>
|
ca_qc_westmount
|
2023-03-01 04:14:28
|
2024-04-27 04:22:07
|
Exception: No email node in b'<div class="vc_row wpb_row vc_row-fluid"><div class="wpb_column vc_column_container vc_col-sm-…
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_qc_westmount/people.py", line 29, in scrape
p.add_contact("email", self.get_email(councillor))
File "/app/scrapers/utils.py", line 141, in get_email
raise Exception("No email node in {}".format(etree.tostring(node)))
Exception: No email node in b'<div class="vc_row wpb_row vc_row-fluid"><div class="wpb_column vc_column_container vc_col-sm-4"><div class="vc_column-inner"><div class="wpb_wrapper">\n\t<div class="wpb_single_image wpb_content_element vc_align_center">\n\t\t\n\t\t<figure class="wpb_wrapper vc_figure">\n\t\t\t<div class="vc_single_image-wrapper vc_box_border_grey"><img loading="lazy" decoding="async" width="310" height="434" src="https://westmount.org/wp-content/uploads/2015/03/christina_m_smith_reg.jpg" class="vc_single_image-img attachment-full" alt="" title="christina_m_smith_reg" srcset="https://westmount.org/wp-content/uploads/2015/03/christina_m_smith_reg.jpg 310w, https://westmount.org/wp-content/uploads/2015/03/christina_m_smith_reg-214x300.jpg 214w" sizes="(max-width: 310px) 100vw, 310px"/></div>\n\t\t</figure>\n\t</div>\n<div class="vc_btn3-container vc_btn3-inline"><a class="vc_general vc_btn3 vc_btn3-size-md vc_btn3-shape-square vc_btn3-style-modern vc_btn3-color-grey" href="https://westmount.org/wp-content/uploads/2015/03/christina_smith_media.jpg" title="" target="_blank">Photo média</a></div></div></div></div><div class="wpb_column vc_column_container vc_col-sm-8"><div class="vc_column-inner"><div class="wpb_wrapper">\n\t<div class="wpb_text_column wpb_content_element ">\n\t\t<div class="wpb_wrapper">\n\t\t\t<h4>Christina M. Smith</h4>\n<p><strong>Mairesse de la Ville de Westmount</strong><br/>\nLa mairesse Christina M. Smith fût élue conseillère municipal en 2013 et mairesse de la Ville de Westmount en 2017 et en 2021.<br/>\nVice-présidente de la Commission de la sécurité publique de l’Agglomération de Montréal.</p>\n<ul>\n<li>Téléphone : <a href="https://westmount.org/conseil-municipal/callto:15149895240">514 989-5240</a></li>\n<li>Télécopieur<a name="patrick-martin"/> : 514 989-5481</li>\n</ul>\n\n\t\t</div>\n\t</div>\n<div class="vc_empty_space" style="height: 20px"><span class="vc_empty_space_inner"/></div><div class="vc_tta-container" data-vc-action="collapseAll"><div class="vc_general vc_tta vc_tta-accordion vc_tta-color-white vc_tta-style-flat vc_tta-shape-square vc_tta-o-shape-group vc_tta-controls-align-left vc_tta-o-all-clickable"><div class="vc_tta-panels-container"><div class="vc_tta-panels"><div class="vc_tta-panel" id="1493140680948-48126fe2-9e0e" data-vc-content=".vc_tta-panel-body"><div class="vc_tta-panel-heading"><h4 class="vc_tta-panel-title vc_tta-controls-icon-position-right"><a href="https://westmount.org/conseil-municipal/#1493140680948-48126fe2-9e0e" data-vc-accordion="" data-vc-container=".vc_tta-container"><span class="vc_tta-title-text">Biographie</span><i class="vc_tta-controls-icon vc_tta-controls-icon-plus"/></a></h4></div><div class="vc_tta-panel-body">\n\t<div class="wpb_text_column wpb_content_element ">\n\t\t<div class="wpb_wrapper">\n\t\t\t<p>Christina M. Smith a été élue au conseil municipal de Westmount le 3 novembre 2013 en tant que conseillère du district 5. Durant ce mandat, elle a été également commissaire à l’administration. En avril 2017, Christina Smith a été nommée mairesse intérimaire de Westmount par le Conseil lorsque le maire Peter Trent a pris sa retraite. Le 5 novembre 2017, Christina Smith a été élue mairesse de la Ville.  Durant son mandat de 2017 à 2021, elle a été membre de la Commission des finances de l’Agglomération de Montréal, de la Commission du développement économique et du Comité d’audit. En novembre 2021, Madame Smith a été réélue mairesse de Westmount pour un second mandat. Elle est également vice-présidente de la Commission de la sécurité publique auprès de l’Agglomération de Montréal. La mairesse Smith est également membre de l’exécutif de l’Association des municipalités de banlieue.</p>\n<p>Avant d’entrer dans la politique municipale, Christina était directrice de l’industrie et des affaires gouvernementales pour Coca-Cola. Elle a également été conseillère politique auprès du ministre fédéral des Affaires intergouvernementales. Impliquée dans de nombreuses organisations communautaires, elle croit à l’engagement civique afin que tous les résidents bénéficient de nombreux programmes et services.</p>\n<p>Christina est titulaire d’un baccalauréat en sciences politiques de l’Université Bishop’s. Elle et son mari ont trois jeunes enfants qui apprécient bon nombre des programmes et activités sportives de Westmount.</p>\n\n\t\t</div>\n\t</div>\n</div></div></div></div></div></div></div></div></div></div>'
|
|
C
|
ca_sk
|
2024-04-27 04:20:22
|
2024-04-27 04:20:22
|
|
D>
04:23:59 WARNING pupa: could not save RunPlan, no successful runs of ocd-jurisdiction/country:ca/csd:4706027/legislature yet
|
ca_sk_regina
|
|
2024-04-27 04:23:59
|
IndexError: list index out of range
Traceback (most recent call last):
File "/app/reports/utils.py", line 71, in scrape_people
report.report = subcommand.handle(args, other)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 260, in handle
return self.do_handle(args, other, juris)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
report['scrape'] = self.do_scrape(juris, args, scrapers)
File "/app/.heroku/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
report[scraper_name] = scraper.do_scrape(**scrape_args)
File "/app/.heroku/src/pupa/pupa/scrape/base.py", line 99, in do_scrape
for obj in self.scrape(**kwargs) or []:
File "/app/scrapers/ca_sk_regina/people.py", line 25, in scrape
yield self.mayor_data(mayor_url)
File "/app/scrapers/ca_sk_regina/people.py", line 61, in mayor_data
phone_path = page.xpath('//div[@class="councillor__contact"]//ul/li/a/@href[contains(., "306")]')[0]
IndexError: list index out of range
|
|
C
|
ca_sk_saskatoon
|
2024-04-27 04:13:06
|
2024-04-27 04:13:06
|
|