CSUHackathon/scrapping_script.py at main · ashisah/CSUHackathon · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
from lxml import html
from bs4 import BeautifulSoup
import re
import requests
import csv

# this is how the data is structred
class Kit:
    def __init__(self, id, dateCreated, currentStatus, statuses):
        self.id=id
        self.dateCreated=dateCreated
        self.currentStatus=currentStatus
        self.statuses=statuses

# make csv file
with open('kits.csv', mode='w') as csvfile:
    fieldnames = ['id', 'date_created', 'current_status', 'status_history']
    writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
    writer.writeheader()

    for x in range(200000,400000):
        # generate URL
        url = "https://sakt.ohioattorneygeneral.gov/timeline?serialNumber="
        i=6-len(str(x))
        ID=""

        for j in range(i):
            ID+='0'

        ID+=str(x)
        url+=ID
        # request html
        page = requests.get(url)
        # parse html
        soup = BeautifulSoup(page.text, 'html.parser')
        tree = html.fromstring(page.content)
        # get list of statuses
        results = soup.find_all('div', attrs={'class':'coc-assignment-tile'})
        statuses=[]

        for result in results:
            result = result.find('div').text
            result=result.strip()
            result=re.sub('\n', '', result)
            result=" ".join(result.split())
            statuses.append(result)

        # get current status
        curStatus = tree.xpath('/html/body/div[1]/div/div[2]/div/div/strong/text()')
        # get date created
        dateCreated = tree.xpath('/html/body/div[1]/div/div[3]/div[1]/div/div/strong/text()')
        # make sure its not a valid kit

        if(len(dateCreated)==0):
            continue

        curStatus=curStatus[0]
        dateCreated=dateCreated[0][8:]
        # store data
        k = Kit(ID, dateCreated, curStatus, statuses)
        writer.writerow({"id":ID, "date_created":dateCreated, "current_status":curStatus, "status_history":statuses})